2024-11-26 23:33:01,046 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 23:33:01,056 main DEBUG Took 0.009068 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 23:33:01,057 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 23:33:01,057 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 23:33:01,058 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 23:33:01,059 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,065 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 23:33:01,078 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,080 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,081 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,083 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,084 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,085 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,088 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,089 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,091 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,092 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:33:01,093 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,093 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 23:33:01,095 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:33:01,097 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 23:33:01,099 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 23:33:01,100 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 23:33:01,102 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 23:33:01,102 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 23:33:01,114 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 23:33:01,117 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 23:33:01,121 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 23:33:01,121 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 23:33:01,122 main DEBUG createAppenders(={Console}) 2024-11-26 23:33:01,123 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-26 23:33:01,123 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 23:33:01,124 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-26 23:33:01,124 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 23:33:01,125 main DEBUG OutputStream closed 2024-11-26 23:33:01,125 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 23:33:01,126 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 23:33:01,126 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-26 23:33:01,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 23:33:01,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 23:33:01,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 23:33:01,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 23:33:01,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 23:33:01,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 23:33:01,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 23:33:01,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 23:33:01,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 23:33:01,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 23:33:01,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 23:33:01,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 23:33:01,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 23:33:01,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 23:33:01,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 23:33:01,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 23:33:01,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 23:33:01,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 23:33:01,276 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 23:33:01,277 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-26 23:33:01,277 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 23:33:01,278 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-26T23:33:01,522 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0 2024-11-26 23:33:01,525 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 23:33:01,525 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T23:33:01,534 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-26T23:33:01,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:33:01,576 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0, deleteOnExit=true 2024-11-26T23:33:01,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:33:01,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/test.cache.data in system properties and HBase conf 2024-11-26T23:33:01,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:33:01,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:33:01,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:33:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:33:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:33:01,708 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T23:33:01,793 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:33:01,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:33:01,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:33:01,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:33:01,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:33:01,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:33:01,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:33:01,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:33:01,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:33:01,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:33:01,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:33:01,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:33:01,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:33:01,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:33:01,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:33:02,747 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T23:33:02,833 INFO [Time-limited test {}] log.Log(170): Logging initialized @2629ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T23:33:02,909 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:33:02,970 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:33:02,997 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:33:02,997 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:33:02,999 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:33:03,013 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:33:03,016 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:33:03,017 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:33:03,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/java.io.tmpdir/jetty-localhost-46519-hadoop-hdfs-3_4_1-tests_jar-_-any-17799061837175577887/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:33:03,273 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:46519} 2024-11-26T23:33:03,273 INFO [Time-limited test {}] server.Server(415): Started @3071ms 2024-11-26T23:33:03,705 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:33:03,712 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:33:03,713 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:33:03,713 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:33:03,713 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:33:03,714 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:33:03,715 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:33:03,834 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/java.io.tmpdir/jetty-localhost-46865-hadoop-hdfs-3_4_1-tests_jar-_-any-6663161291811476325/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:33:03,834 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:46865} 2024-11-26T23:33:03,835 INFO [Time-limited test {}] server.Server(415): Started @3632ms 2024-11-26T23:33:03,887 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:33:04,383 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0/data/data2/current/BP-1290680769-172.17.0.2-1732663982490/current, will proceed with Du for space computation calculation, 2024-11-26T23:33:04,383 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0/data/data1/current/BP-1290680769-172.17.0.2-1732663982490/current, will proceed with Du for space computation calculation, 2024-11-26T23:33:04,438 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:33:04,508 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5ba84c53b24366a with lease ID 0x8aac9a7d7aead21c: Processing first storage report for DS-88544e95-2d8a-474b-8bed-b8c806623367 from datanode DatanodeRegistration(127.0.0.1:36035, datanodeUuid=6dcbde4b-1ee6-4b73-8737-d9c1a29e5185, infoPort=38249, infoSecurePort=0, ipcPort=42255, storageInfo=lv=-57;cid=testClusterID;nsid=657332300;c=1732663982490) 2024-11-26T23:33:04,509 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5ba84c53b24366a with lease ID 0x8aac9a7d7aead21c: from storage DS-88544e95-2d8a-474b-8bed-b8c806623367 node DatanodeRegistration(127.0.0.1:36035, datanodeUuid=6dcbde4b-1ee6-4b73-8737-d9c1a29e5185, infoPort=38249, infoSecurePort=0, ipcPort=42255, storageInfo=lv=-57;cid=testClusterID;nsid=657332300;c=1732663982490), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T23:33:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5ba84c53b24366a with lease ID 0x8aac9a7d7aead21c: Processing first storage report for DS-2cd04791-d12f-41f9-9e14-f6091135cc1d from datanode DatanodeRegistration(127.0.0.1:36035, datanodeUuid=6dcbde4b-1ee6-4b73-8737-d9c1a29e5185, infoPort=38249, infoSecurePort=0, ipcPort=42255, storageInfo=lv=-57;cid=testClusterID;nsid=657332300;c=1732663982490) 2024-11-26T23:33:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5ba84c53b24366a with lease ID 0x8aac9a7d7aead21c: from storage DS-2cd04791-d12f-41f9-9e14-f6091135cc1d node DatanodeRegistration(127.0.0.1:36035, datanodeUuid=6dcbde4b-1ee6-4b73-8737-d9c1a29e5185, infoPort=38249, infoSecurePort=0, ipcPort=42255, storageInfo=lv=-57;cid=testClusterID;nsid=657332300;c=1732663982490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:33:04,593 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0 2024-11-26T23:33:04,690 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0/zookeeper_0, clientPort=60864, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:33:04,701 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60864 2024-11-26T23:33:04,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:33:04,722 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:33:05,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:33:05,435 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd with version=8 2024-11-26T23:33:05,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/hbase-staging 2024-11-26T23:33:05,552 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T23:33:05,855 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:33:05,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:33:05,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:33:05,874 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:33:05,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:33:05,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:33:06,061 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:33:06,149 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T23:33:06,162 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T23:33:06,167 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:33:06,205 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 18621 (auto-detected) 2024-11-26T23:33:06,206 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T23:33:06,231 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35833 2024-11-26T23:33:06,263 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35833 connecting to ZooKeeper ensemble=127.0.0.1:60864 2024-11-26T23:33:06,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358330x0, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:33:06,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35833-0x10156a02cac0000 connected 2024-11-26T23:33:06,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:33:06,393 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:33:06,406 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:33:06,411 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd, hbase.cluster.distributed=false 2024-11-26T23:33:06,442 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:33:06,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-26T23:33:06,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35833 2024-11-26T23:33:06,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35833 2024-11-26T23:33:06,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-26T23:33:06,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-26T23:33:06,579 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:33:06,582 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:33:06,582 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:33:06,582 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:33:06,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:33:06,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:33:06,587 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:33:06,590 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:33:06,591 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43077 2024-11-26T23:33:06,593 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43077 connecting to ZooKeeper ensemble=127.0.0.1:60864 2024-11-26T23:33:06,595 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:33:06,599 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:33:06,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430770x0, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:33:06,609 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430770x0, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:33:06,609 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43077-0x10156a02cac0001 connected 2024-11-26T23:33:06,613 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:33:06,624 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:33:06,627 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:33:06,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:33:06,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43077 2024-11-26T23:33:06,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43077 2024-11-26T23:33:06,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43077 2024-11-26T23:33:06,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43077 2024-11-26T23:33:06,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43077 2024-11-26T23:33:06,649 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:35833 2024-11-26T23:33:06,651 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,35833,1732663985624 2024-11-26T23:33:06,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:33:06,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:33:06,660 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,35833,1732663985624 2024-11-26T23:33:06,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:33:06,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:06,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:06,691 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:33:06,692 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,35833,1732663985624 from backup master directory 2024-11-26T23:33:06,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,35833,1732663985624 2024-11-26T23:33:06,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:33:06,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:33:06,695 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:33:06,696 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,35833,1732663985624 2024-11-26T23:33:06,698 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T23:33:06,701 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T23:33:06,760 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/hbase.id] with ID: 8c4ad21b-9223-4523-b167-1f3f7b98c684 2024-11-26T23:33:06,760 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/.tmp/hbase.id 2024-11-26T23:33:06,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:33:07,173 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/.tmp/hbase.id]:[hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/hbase.id] 2024-11-26T23:33:07,221 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:33:07,226 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:33:07,246 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-26T23:33:07,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:07,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:07,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:33:07,686 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:33:07,690 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:33:07,707 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:07,712 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:33:07,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:33:08,169 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store 2024-11-26T23:33:08,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:33:08,203 INFO [master/c707e9ace675:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T23:33:08,207 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:08,209 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:33:08,209 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:33:08,210 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:33:08,212 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:33:08,212 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:33:08,212 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:33:08,214 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663988209Disabling compacts and flushes for region at 1732663988209Disabling writes for close at 1732663988212 (+3 ms)Writing region close event to WAL at 1732663988212Closed at 1732663988212 2024-11-26T23:33:08,218 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/.initializing 2024-11-26T23:33:08,218 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/WALs/c707e9ace675,35833,1732663985624 2024-11-26T23:33:08,234 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:33:08,256 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C35833%2C1732663985624, suffix=, logDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/WALs/c707e9ace675,35833,1732663985624, archiveDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/oldWALs, maxLogs=10 2024-11-26T23:33:08,288 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/WALs/c707e9ace675,35833,1732663985624/c707e9ace675%2C35833%2C1732663985624.1732663988262, exclude list is [], retry=0 2024-11-26T23:33:08,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36035,DS-88544e95-2d8a-474b-8bed-b8c806623367,DISK] 2024-11-26T23:33:08,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T23:33:08,362 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/WALs/c707e9ace675,35833,1732663985624/c707e9ace675%2C35833%2C1732663985624.1732663988262 2024-11-26T23:33:08,364 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38249:38249)] 2024-11-26T23:33:08,364 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:33:08,365 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:08,370 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,371 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:33:08,461 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:08,464 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:08,465 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:33:08,469 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:08,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:08,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:33:08,474 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:08,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:08,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:33:08,478 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:08,479 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:08,480 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,483 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,484 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,489 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,490 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,493 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:33:08,497 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:33:08,502 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:33:08,503 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73143611, jitterRate=0.08992473781108856}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:33:08,510 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663988387Initializing all the Stores at 1732663988391 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663988392 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663988392Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663988393 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663988393Cleaning up temporary data from old regions at 1732663988490 (+97 ms)Region opened successfully at 1732663988510 (+20 ms) 2024-11-26T23:33:08,512 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:33:08,553 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5c6789, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:33:08,588 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:33:08,600 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:33:08,600 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:33:08,604 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:33:08,605 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T23:33:08,611 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T23:33:08,611 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:33:08,638 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:33:08,649 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:33:08,652 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:33:08,655 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:33:08,658 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:33:08,660 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:33:08,662 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:33:08,666 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:33:08,668 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:33:08,669 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:33:08,672 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:33:08,694 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:33:08,695 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:33:08,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:33:08,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:33:08,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:08,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:08,705 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,35833,1732663985624, sessionid=0x10156a02cac0000, setting cluster-up flag (Was=false) 2024-11-26T23:33:08,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:08,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:08,732 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:33:08,734 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,35833,1732663985624 2024-11-26T23:33:08,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:08,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:08,752 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:33:08,754 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,35833,1732663985624 2024-11-26T23:33:08,763 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:33:08,840 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(746): ClusterId : 8c4ad21b-9223-4523-b167-1f3f7b98c684 2024-11-26T23:33:08,842 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:33:08,844 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:33:08,847 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:33:08,847 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:33:08,851 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:33:08,852 DEBUG [RS:0;c707e9ace675:43077 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62364aeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:33:08,855 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:33:08,862 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:33:08,868 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:43077 2024-11-26T23:33:08,867 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,35833,1732663985624 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:33:08,871 INFO [RS:0;c707e9ace675:43077 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:33:08,871 INFO [RS:0;c707e9ace675:43077 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:33:08,871 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:33:08,874 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,35833,1732663985624 with port=43077, startcode=1732663986534 2024-11-26T23:33:08,876 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:33:08,876 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:33:08,876 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:33:08,876 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:33:08,877 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:33:08,877 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:08,877 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:33:08,878 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:08,882 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732664018882 2024-11-26T23:33:08,884 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:33:08,885 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:33:08,886 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:33:08,886 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:33:08,888 DEBUG [RS:0;c707e9ace675:43077 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:33:08,890 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:33:08,890 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:33:08,891 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:33:08,891 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:33:08,894 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:08,892 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:08,894 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:33:08,896 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:33:08,898 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:33:08,898 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:33:08,901 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:33:08,901 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:33:08,903 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663988902,5,FailOnTimeoutGroup] 2024-11-26T23:33:08,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:33:08,908 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663988903,5,FailOnTimeoutGroup] 2024-11-26T23:33:08,908 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:08,909 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:33:08,910 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:08,911 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:08,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46245, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:33:08,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35833 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,43077,1732663986534 2024-11-26T23:33:08,937 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35833 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,43077,1732663986534 2024-11-26T23:33:08,951 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:33:08,951 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36065 2024-11-26T23:33:08,952 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:33:08,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:33:08,956 DEBUG [RS:0;c707e9ace675:43077 {}] zookeeper.ZKUtil(111): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,43077,1732663986534 2024-11-26T23:33:08,957 WARN [RS:0;c707e9ace675:43077 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:33:08,957 INFO [RS:0;c707e9ace675:43077 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:33:08,957 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/WALs/c707e9ace675,43077,1732663986534 2024-11-26T23:33:08,959 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,43077,1732663986534] 2024-11-26T23:33:08,982 INFO [RS:0;c707e9ace675:43077 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:33:08,995 INFO [RS:0;c707e9ace675:43077 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:33:09,000 INFO [RS:0;c707e9ace675:43077 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:33:09,000 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,001 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:33:09,007 INFO [RS:0;c707e9ace675:43077 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:33:09,008 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,008 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,009 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,009 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,009 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,009 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,009 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:33:09,009 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,010 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,010 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,010 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,010 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,010 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:33:09,010 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:33:09,010 DEBUG [RS:0;c707e9ace675:43077 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:33:09,011 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,012 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,012 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,012 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,012 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,012 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,43077,1732663986534-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:33:09,033 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:33:09,035 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,43077,1732663986534-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,036 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,036 INFO [RS:0;c707e9ace675:43077 {}] regionserver.Replication(171): c707e9ace675,43077,1732663986534 started 2024-11-26T23:33:09,055 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:09,056 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,43077,1732663986534, RpcServer on c707e9ace675/172.17.0.2:43077, sessionid=0x10156a02cac0001 2024-11-26T23:33:09,057 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:33:09,057 DEBUG [RS:0;c707e9ace675:43077 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,43077,1732663986534 2024-11-26T23:33:09,057 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,43077,1732663986534' 2024-11-26T23:33:09,057 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:33:09,058 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:33:09,059 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:33:09,059 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:33:09,059 DEBUG [RS:0;c707e9ace675:43077 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,43077,1732663986534 2024-11-26T23:33:09,059 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,43077,1732663986534' 2024-11-26T23:33:09,059 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:33:09,060 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:33:09,060 DEBUG [RS:0;c707e9ace675:43077 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:33:09,061 INFO [RS:0;c707e9ace675:43077 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:33:09,061 INFO [RS:0;c707e9ace675:43077 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:33:09,166 INFO [RS:0;c707e9ace675:43077 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:33:09,170 INFO [RS:0;c707e9ace675:43077 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C43077%2C1732663986534, suffix=, logDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/WALs/c707e9ace675,43077,1732663986534, archiveDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/oldWALs, maxLogs=32 2024-11-26T23:33:09,187 DEBUG [RS:0;c707e9ace675:43077 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/WALs/c707e9ace675,43077,1732663986534/c707e9ace675%2C43077%2C1732663986534.1732663989172, exclude list is [], retry=0 2024-11-26T23:33:09,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36035,DS-88544e95-2d8a-474b-8bed-b8c806623367,DISK] 2024-11-26T23:33:09,195 INFO [RS:0;c707e9ace675:43077 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/WALs/c707e9ace675,43077,1732663986534/c707e9ace675%2C43077%2C1732663986534.1732663989172 2024-11-26T23:33:09,196 DEBUG [RS:0;c707e9ace675:43077 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38249:38249)] 2024-11-26T23:33:09,309 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:33:09,309 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:33:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741833_1009 (size=32) 2024-11-26T23:33:09,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:09,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:33:09,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:33:09,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:09,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:09,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:33:09,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:33:09,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:09,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:09,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:33:09,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:33:09,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:09,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:09,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:33:09,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:33:09,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:09,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:09,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:33:09,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740 2024-11-26T23:33:09,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740 2024-11-26T23:33:09,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:33:09,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:33:09,751 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:33:09,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:33:09,758 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:33:09,759 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59452039, jitterRate=-0.11409558355808258}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:33:09,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663989724Initializing all the Stores at 1732663989726 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663989726Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663989726Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663989726Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663989726Cleaning up temporary data from old regions at 1732663989750 (+24 ms)Region opened successfully at 1732663989761 (+11 ms) 2024-11-26T23:33:09,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:33:09,762 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:33:09,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:33:09,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:33:09,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:33:09,763 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:33:09,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663989762Disabling compacts and flushes for region at 1732663989762Disabling writes for close at 1732663989762Writing region close event to WAL at 1732663989763 (+1 ms)Closed at 1732663989763 2024-11-26T23:33:09,767 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:33:09,767 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:33:09,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:33:09,783 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:33:09,786 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:33:09,939 DEBUG [c707e9ace675:35833 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:33:09,953 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:09,961 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,43077,1732663986534, state=OPENING 2024-11-26T23:33:09,969 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:33:09,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:09,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:33:09,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:33:09,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:33:09,974 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:33:09,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:33:10,154 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:33:10,157 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50571, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:33:10,169 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:33:10,170 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:33:10,170 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T23:33:10,174 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C43077%2C1732663986534.meta, suffix=.meta, logDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/WALs/c707e9ace675,43077,1732663986534, archiveDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/oldWALs, maxLogs=32 2024-11-26T23:33:10,190 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/WALs/c707e9ace675,43077,1732663986534/c707e9ace675%2C43077%2C1732663986534.meta.1732663990175.meta, exclude list is [], retry=0 2024-11-26T23:33:10,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36035,DS-88544e95-2d8a-474b-8bed-b8c806623367,DISK] 2024-11-26T23:33:10,198 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/WALs/c707e9ace675,43077,1732663986534/c707e9ace675%2C43077%2C1732663986534.meta.1732663990175.meta 2024-11-26T23:33:10,198 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38249:38249)] 2024-11-26T23:33:10,199 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:33:10,200 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:33:10,204 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:33:10,208 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:33:10,213 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:33:10,214 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:10,214 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:33:10,214 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:33:10,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:33:10,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:33:10,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:10,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:10,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:33:10,222 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:33:10,223 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:10,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:10,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:33:10,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:33:10,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:10,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:10,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:33:10,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:33:10,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:10,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:33:10,228 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:33:10,230 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740 2024-11-26T23:33:10,232 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740 2024-11-26T23:33:10,235 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:33:10,235 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:33:10,236 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:33:10,239 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:33:10,241 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64474382, jitterRate=-0.03925684094429016}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:33:10,241 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:33:10,244 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663990215Writing region info on filesystem at 1732663990215Initializing all the Stores at 1732663990217 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663990217Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663990218 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663990218Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663990218Cleaning up temporary data from old regions at 1732663990235 (+17 ms)Running coprocessor post-open hooks at 1732663990241 (+6 ms)Region opened successfully at 1732663990243 (+2 ms) 2024-11-26T23:33:10,253 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663990145 2024-11-26T23:33:10,267 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:33:10,267 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:33:10,269 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:10,272 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,43077,1732663986534, state=OPEN 2024-11-26T23:33:10,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:33:10,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:33:10,278 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:33:10,278 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:33:10,278 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:10,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:33:10,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,43077,1732663986534 in 302 msec 2024-11-26T23:33:10,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:33:10,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 513 msec 2024-11-26T23:33:10,294 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:33:10,294 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:33:10,315 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:10,316 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:10,338 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:10,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:10,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5700 sec 2024-11-26T23:33:10,369 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663990369, completionTime=-1 2024-11-26T23:33:10,372 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:33:10,372 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:33:10,421 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:33:10,421 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732664050421 2024-11-26T23:33:10,421 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732664110421 2024-11-26T23:33:10,421 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 49 msec 2024-11-26T23:33:10,424 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35833,1732663985624-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:10,424 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35833,1732663985624-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:10,424 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35833,1732663985624-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:10,426 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:35833, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:10,426 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:10,427 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:10,437 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:33:10,461 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.765sec 2024-11-26T23:33:10,462 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:33:10,463 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:33:10,465 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:33:10,465 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:33:10,465 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:33:10,466 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35833,1732663985624-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:33:10,467 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35833,1732663985624-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:33:10,476 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:33:10,476 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:33:10,477 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35833,1732663985624-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:33:10,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4346ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:10,553 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T23:33:10,553 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T23:33:10,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:10,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:10,566 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5246c284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:10,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:10,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbb1cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:10,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:10,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:10,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,35833,1732663985624 2024-11-26T23:33:10,619 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=7450 2024-11-26T23:33:10,628 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:33:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35833,1732663985624 2024-11-26T23:33:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23562587 2024-11-26T23:33:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:33:10,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:33:10,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:33:10,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:33:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:10,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:33:10,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-26T23:33:10,659 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:10,661 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:33:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:33:10,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741835_1011 (size=959) 2024-11-26T23:33:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:33:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:33:11,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:33:11,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741836_1012 (size=53) 2024-11-26T23:33:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:33:11,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:11,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 56adae88c925ec88c99616a57d431fab, disabling compactions & flushes 2024-11-26T23:33:11,486 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:11,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:11,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. after waiting 0 ms 2024-11-26T23:33:11,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:11,486 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:11,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56adae88c925ec88c99616a57d431fab: Waiting for close lock at 1732663991485Disabling compacts and flushes for region at 1732663991485Disabling writes for close at 1732663991486 (+1 ms)Writing region close event to WAL at 1732663991486Closed at 1732663991486 2024-11-26T23:33:11,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:33:11,494 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732663991489"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663991489"}]},"ts":"1732663991489"} 2024-11-26T23:33:11,499 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:33:11,501 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:33:11,503 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663991501"}]},"ts":"1732663991501"} 2024-11-26T23:33:11,507 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T23:33:11,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, ASSIGN}] 2024-11-26T23:33:11,511 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, ASSIGN 2024-11-26T23:33:11,513 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, ASSIGN; state=OFFLINE, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=false 2024-11-26T23:33:11,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=56adae88c925ec88c99616a57d431fab, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:11,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, ASSIGN because future has completed 2024-11-26T23:33:11,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:33:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:33:11,832 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:11,832 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:33:11,833 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,833 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:11,833 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,833 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,835 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,838 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56adae88c925ec88c99616a57d431fab columnFamilyName A 2024-11-26T23:33:11,838 DEBUG [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:11,839 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] regionserver.HStore(327): Store=56adae88c925ec88c99616a57d431fab/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:11,839 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,841 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56adae88c925ec88c99616a57d431fab columnFamilyName B 2024-11-26T23:33:11,841 DEBUG [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:11,842 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] regionserver.HStore(327): Store=56adae88c925ec88c99616a57d431fab/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:11,842 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,844 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56adae88c925ec88c99616a57d431fab columnFamilyName C 2024-11-26T23:33:11,844 DEBUG [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:11,845 INFO [StoreOpener-56adae88c925ec88c99616a57d431fab-1 {}] regionserver.HStore(327): Store=56adae88c925ec88c99616a57d431fab/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:11,845 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,846 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,847 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,849 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,849 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,850 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:33:11,852 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,855 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:33:11,856 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 56adae88c925ec88c99616a57d431fab; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63471117, jitterRate=-0.05420665442943573}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:33:11,856 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:11,857 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 56adae88c925ec88c99616a57d431fab: Running coprocessor pre-open hook at 1732663991833Writing region info on filesystem at 1732663991833Initializing all the Stores at 1732663991835 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663991835Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663991835Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663991835Cleaning up temporary data from old regions at 1732663991849 (+14 ms)Running coprocessor post-open hooks at 1732663991856 (+7 ms)Region opened successfully at 1732663991857 (+1 ms) 2024-11-26T23:33:11,859 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., pid=6, masterSystemTime=1732663991825 2024-11-26T23:33:11,863 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:11,863 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:11,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=56adae88c925ec88c99616a57d431fab, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:11,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:33:11,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T23:33:11,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 in 198 msec 2024-11-26T23:33:11,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T23:33:11,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, ASSIGN in 364 msec 2024-11-26T23:33:11,878 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:33:11,878 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663991878"}]},"ts":"1732663991878"} 2024-11-26T23:33:11,881 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T23:33:11,882 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:33:11,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2330 sec 2024-11-26T23:33:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:33:12,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:12,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,832 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d34895b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50806732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,837 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,844 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:33:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35833,1732663985624 2024-11-26T23:33:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d4a6a92 2024-11-26T23:33:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:33:12,849 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:33:12,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,855 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3728ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f8ac40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,861 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14496fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e49c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,868 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e59e9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,873 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9c1aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68da59ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,881 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d38fddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bbabde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,886 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b71435d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6d26ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,891 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79aeff1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64994b9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,896 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f551be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,899 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5180f463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c9a155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:12,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:12,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:12,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4770a9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:12,909 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,909 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,909 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,909 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,911 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,914 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,923 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-26T23:33:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,928 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,931 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:12,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:12,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,938 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:12,957 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:33:12,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:33:13,036 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:13,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:33:13,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/203a3fe210eb49879534e68a3508ec1f is 50, key is test_row_0/A:col10/1732663993005/Put/seqid=0 2024-11-26T23:33:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741837_1013 (size=12001) 2024-11-26T23:33:13,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/203a3fe210eb49879534e68a3508ec1f 2024-11-26T23:33:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664053219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664053217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664053250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664053253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664053256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:33:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:13,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/65c0f4c303284d3f885e775b7f6f1b6d is 50, key is test_row_0/B:col10/1732663993005/Put/seqid=0 2024-11-26T23:33:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741838_1014 (size=12001) 2024-11-26T23:33:13,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/65c0f4c303284d3f885e775b7f6f1b6d 2024-11-26T23:33:13,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/a0d975266a2d4101b79187263483ac7e is 50, key is test_row_0/C:col10/1732663993005/Put/seqid=0 2024-11-26T23:33:13,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741839_1015 (size=12001) 2024-11-26T23:33:13,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664053423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664053423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664053424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664053423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664053431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:33:13,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664053632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664053633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664053644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664053644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664053644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:13,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/a0d975266a2d4101b79187263483ac7e 2024-11-26T23:33:13,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/203a3fe210eb49879534e68a3508ec1f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/203a3fe210eb49879534e68a3508ec1f 2024-11-26T23:33:13,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/203a3fe210eb49879534e68a3508ec1f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:33:13,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/65c0f4c303284d3f885e775b7f6f1b6d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/65c0f4c303284d3f885e775b7f6f1b6d 2024-11-26T23:33:13,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/65c0f4c303284d3f885e775b7f6f1b6d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:33:13,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/a0d975266a2d4101b79187263483ac7e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a0d975266a2d4101b79187263483ac7e 2024-11-26T23:33:13,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a0d975266a2d4101b79187263483ac7e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:33:13,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 56adae88c925ec88c99616a57d431fab in 860ms, sequenceid=13, compaction requested=false 2024-11-26T23:33:13,903 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T23:33:13,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:13,960 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:13,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T23:33:13,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:13,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:13,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8615b0a309e145ec909009896cdedad4 is 50, key is test_row_0/A:col10/1732663993240/Put/seqid=0 2024-11-26T23:33:13,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741840_1016 (size=14341) 2024-11-26T23:33:14,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8615b0a309e145ec909009896cdedad4 2024-11-26T23:33:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664053978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664053980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664053990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664053986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664053994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/dc6c26f64a2a4606b94ccbb11beedf48 is 50, key is test_row_0/B:col10/1732663993240/Put/seqid=0 2024-11-26T23:33:14,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741841_1017 (size=12001) 2024-11-26T23:33:14,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/dc6c26f64a2a4606b94ccbb11beedf48 2024-11-26T23:33:14,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/f78a9da310af42c189f5763b1bc5db2a is 50, key is test_row_0/C:col10/1732663993240/Put/seqid=0 2024-11-26T23:33:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:33:14,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741842_1018 (size=12001) 2024-11-26T23:33:14,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:14,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:14,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664054112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664054123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664054124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664054125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664054134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:14,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:14,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:14,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664054343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664054343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664054353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664054354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664054354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:14,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:14,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/f78a9da310af42c189f5763b1bc5db2a 2024-11-26T23:33:14,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8615b0a309e145ec909009896cdedad4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8615b0a309e145ec909009896cdedad4 2024-11-26T23:33:14,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8615b0a309e145ec909009896cdedad4, entries=200, sequenceid=39, filesize=14.0 K 2024-11-26T23:33:14,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/dc6c26f64a2a4606b94ccbb11beedf48 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dc6c26f64a2a4606b94ccbb11beedf48 2024-11-26T23:33:14,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dc6c26f64a2a4606b94ccbb11beedf48, entries=150, sequenceid=39, filesize=11.7 K 2024-11-26T23:33:14,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/f78a9da310af42c189f5763b1bc5db2a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f78a9da310af42c189f5763b1bc5db2a 2024-11-26T23:33:14,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f78a9da310af42c189f5763b1bc5db2a, entries=150, sequenceid=39, filesize=11.7 K 2024-11-26T23:33:14,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 56adae88c925ec88c99616a57d431fab in 629ms, sequenceid=39, compaction requested=false 2024-11-26T23:33:14,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:14,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:33:14,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:14,612 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T23:33:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/134f125058e44da79375e666c1664414 is 50, key is test_row_0/A:col10/1732663993989/Put/seqid=0 2024-11-26T23:33:14,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741843_1019 (size=12001) 2024-11-26T23:33:14,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/134f125058e44da79375e666c1664414 2024-11-26T23:33:14,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:14,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d41c80ab4d254bb6809f5d09ed92522b is 50, key is test_row_0/B:col10/1732663993989/Put/seqid=0 2024-11-26T23:33:14,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741844_1020 (size=12001) 2024-11-26T23:33:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664054720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664054732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664054732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664054735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664054735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664054842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664054852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664054852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664054852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664054853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:14,980 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:33:15,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664055052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664055062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664055063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664055063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664055063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,091 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d41c80ab4d254bb6809f5d09ed92522b 2024-11-26T23:33:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/36a99eb252004fb08f7462be4edbe02d is 50, key is test_row_0/C:col10/1732663993989/Put/seqid=0 2024-11-26T23:33:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:33:15,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741845_1021 (size=12001) 2024-11-26T23:33:15,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664055362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664055372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664055372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664055373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664055373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,537 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/36a99eb252004fb08f7462be4edbe02d 2024-11-26T23:33:15,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/134f125058e44da79375e666c1664414 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/134f125058e44da79375e666c1664414 2024-11-26T23:33:15,574 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/134f125058e44da79375e666c1664414, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T23:33:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d41c80ab4d254bb6809f5d09ed92522b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d41c80ab4d254bb6809f5d09ed92522b 2024-11-26T23:33:15,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d41c80ab4d254bb6809f5d09ed92522b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T23:33:15,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/36a99eb252004fb08f7462be4edbe02d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/36a99eb252004fb08f7462be4edbe02d 2024-11-26T23:33:15,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/36a99eb252004fb08f7462be4edbe02d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T23:33:15,609 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 56adae88c925ec88c99616a57d431fab in 996ms, sequenceid=49, compaction requested=true 2024-11-26T23:33:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T23:33:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T23:33:15,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T23:33:15,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6790 sec 2024-11-26T23:33:15,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7030 sec 2024-11-26T23:33:15,725 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:33:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:15,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:15,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:33:15,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/b938a5707f484fa7b338ef4daa003035 is 50, key is test_row_0/A:col10/1732663994726/Put/seqid=0 2024-11-26T23:33:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664055889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664055890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664055893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664055893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664055898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:15,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741846_1022 (size=14341) 2024-11-26T23:33:16,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664056002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664056002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664056012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664056012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664056013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,145 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:33:16,146 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T23:33:16,148 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:33:16,148 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T23:33:16,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T23:33:16,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T23:33:16,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T23:33:16,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T23:33:16,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664056212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664056213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664056222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664056222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664056222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/b938a5707f484fa7b338ef4daa003035 2024-11-26T23:33:16,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/0f320b90f2d24672bc36927fb46d22aa is 50, key is test_row_0/B:col10/1732663994726/Put/seqid=0 2024-11-26T23:33:16,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741847_1023 (size=12001) 2024-11-26T23:33:16,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/0f320b90f2d24672bc36927fb46d22aa 2024-11-26T23:33:16,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/38539fdd26544b988076c583a59e3b77 is 50, key is test_row_0/C:col10/1732663994726/Put/seqid=0 2024-11-26T23:33:16,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741848_1024 (size=12001) 2024-11-26T23:33:16,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/38539fdd26544b988076c583a59e3b77 2024-11-26T23:33:16,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/b938a5707f484fa7b338ef4daa003035 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b938a5707f484fa7b338ef4daa003035 2024-11-26T23:33:16,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b938a5707f484fa7b338ef4daa003035, entries=200, sequenceid=76, filesize=14.0 K 2024-11-26T23:33:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-26T23:33:16,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/0f320b90f2d24672bc36927fb46d22aa as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/0f320b90f2d24672bc36927fb46d22aa 2024-11-26T23:33:16,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/0f320b90f2d24672bc36927fb46d22aa, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T23:33:16,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/38539fdd26544b988076c583a59e3b77 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/38539fdd26544b988076c583a59e3b77 2024-11-26T23:33:16,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/38539fdd26544b988076c583a59e3b77, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T23:33:16,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 56adae88c925ec88c99616a57d431fab in 579ms, sequenceid=76, compaction requested=true 2024-11-26T23:33:16,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:16,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:16,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:16,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:16,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:16,466 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:16,466 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:16,474 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:16,474 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:16,476 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:16,477 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:16,477 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/203a3fe210eb49879534e68a3508ec1f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8615b0a309e145ec909009896cdedad4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/134f125058e44da79375e666c1664414, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b938a5707f484fa7b338ef4daa003035] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=51.4 K 2024-11-26T23:33:16,479 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:16,479 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:16,479 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/65c0f4c303284d3f885e775b7f6f1b6d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dc6c26f64a2a4606b94ccbb11beedf48, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d41c80ab4d254bb6809f5d09ed92522b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/0f320b90f2d24672bc36927fb46d22aa] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=46.9 K 2024-11-26T23:33:16,481 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c0f4c303284d3f885e775b7f6f1b6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732663993005 2024-11-26T23:33:16,481 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203a3fe210eb49879534e68a3508ec1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732663993005 2024-11-26T23:33:16,482 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting dc6c26f64a2a4606b94ccbb11beedf48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732663993210 2024-11-26T23:33:16,482 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8615b0a309e145ec909009896cdedad4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732663993210 2024-11-26T23:33:16,483 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d41c80ab4d254bb6809f5d09ed92522b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732663993965 2024-11-26T23:33:16,483 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 134f125058e44da79375e666c1664414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732663993965 2024-11-26T23:33:16,484 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f320b90f2d24672bc36927fb46d22aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732663994726 2024-11-26T23:33:16,486 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b938a5707f484fa7b338ef4daa003035, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732663994726 2024-11-26T23:33:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:16,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:16,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:16,547 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#12 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:16,547 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#13 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:16,549 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/03e5d365ffff42f6831acd7082e5276e is 50, key is test_row_0/A:col10/1732663994726/Put/seqid=0 2024-11-26T23:33:16,549 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/de7c5382df70452d8bea775ab7288eb1 is 50, key is test_row_0/B:col10/1732663994726/Put/seqid=0 2024-11-26T23:33:16,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/539a5157586544acae7d6f61392077bf is 50, key is test_row_0/A:col10/1732663995888/Put/seqid=0 2024-11-26T23:33:16,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741850_1026 (size=12139) 2024-11-26T23:33:16,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741849_1025 (size=12139) 2024-11-26T23:33:16,606 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/03e5d365ffff42f6831acd7082e5276e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/03e5d365ffff42f6831acd7082e5276e 2024-11-26T23:33:16,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741851_1027 (size=12001) 2024-11-26T23:33:16,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/539a5157586544acae7d6f61392077bf 2024-11-26T23:33:16,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d60042642a7e4c509d4a46a3602a0d92 is 50, key is test_row_0/B:col10/1732663995888/Put/seqid=0 2024-11-26T23:33:16,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741852_1028 (size=12001) 2024-11-26T23:33:16,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664056631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664056627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,693 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into 03e5d365ffff42f6831acd7082e5276e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:16,693 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:16,696 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=12, startTime=1732663996457; duration=0sec 2024-11-26T23:33:16,697 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:16,697 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:16,697 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:16,700 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:16,700 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:16,700 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:16,701 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a0d975266a2d4101b79187263483ac7e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f78a9da310af42c189f5763b1bc5db2a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/36a99eb252004fb08f7462be4edbe02d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/38539fdd26544b988076c583a59e3b77] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=46.9 K 2024-11-26T23:33:16,703 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d975266a2d4101b79187263483ac7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732663993005 2024-11-26T23:33:16,703 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f78a9da310af42c189f5763b1bc5db2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732663993210 2024-11-26T23:33:16,704 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a99eb252004fb08f7462be4edbe02d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732663993965 2024-11-26T23:33:16,705 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38539fdd26544b988076c583a59e3b77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732663994726 2024-11-26T23:33:16,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664056690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664056690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664056692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,732 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#16 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:16,734 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d0770b46a436454abf95196ab3a70931 is 50, key is test_row_0/C:col10/1732663994726/Put/seqid=0 2024-11-26T23:33:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741853_1029 (size=12139) 2024-11-26T23:33:16,779 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d0770b46a436454abf95196ab3a70931 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d0770b46a436454abf95196ab3a70931 2024-11-26T23:33:16,796 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into d0770b46a436454abf95196ab3a70931(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:16,796 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:16,797 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=12, startTime=1732663996463; duration=0sec 2024-11-26T23:33:16,797 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:16,797 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:16,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664056802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664056802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664056812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664056822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664056822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,004 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/de7c5382df70452d8bea775ab7288eb1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/de7c5382df70452d8bea775ab7288eb1 2024-11-26T23:33:17,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664057011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664057012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,026 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into de7c5382df70452d8bea775ab7288eb1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:17,026 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:17,026 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=12, startTime=1732663996463; duration=0sec 2024-11-26T23:33:17,026 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:17,026 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664057031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664057032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664057032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d60042642a7e4c509d4a46a3602a0d92 2024-11-26T23:33:17,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/61e15f046c664e858726dd5fedba9c0d is 50, key is test_row_0/C:col10/1732663995888/Put/seqid=0 2024-11-26T23:33:17,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741854_1030 (size=12001) 2024-11-26T23:33:17,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/61e15f046c664e858726dd5fedba9c0d 2024-11-26T23:33:17,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/539a5157586544acae7d6f61392077bf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/539a5157586544acae7d6f61392077bf 2024-11-26T23:33:17,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/539a5157586544acae7d6f61392077bf, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T23:33:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d60042642a7e4c509d4a46a3602a0d92 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d60042642a7e4c509d4a46a3602a0d92 2024-11-26T23:33:17,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d60042642a7e4c509d4a46a3602a0d92, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T23:33:17,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/61e15f046c664e858726dd5fedba9c0d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/61e15f046c664e858726dd5fedba9c0d 2024-11-26T23:33:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:33:17,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:17,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-26T23:33:17,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/61e15f046c664e858726dd5fedba9c0d, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T23:33:17,140 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T23:33:17,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 56adae88c925ec88c99616a57d431fab in 606ms, sequenceid=87, compaction requested=false 2024-11-26T23:33:17,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:17,142 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:17,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T23:33:17,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T23:33:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:17,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:33:17,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/0d9c430f711e4b708e3a2f2b39187020 is 50, key is test_row_0/A:col10/1732663996656/Put/seqid=0 2024-11-26T23:33:17,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741855_1031 (size=12001) 2024-11-26T23:33:17,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/0d9c430f711e4b708e3a2f2b39187020 2024-11-26T23:33:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:17,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/06b845f8fd3a4cd9a4b2462ac1bba2fb is 50, key is test_row_0/B:col10/1732663996656/Put/seqid=0 2024-11-26T23:33:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664057350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741856_1032 (size=12001) 2024-11-26T23:33:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664057350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664057352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664057355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/06b845f8fd3a4cd9a4b2462ac1bba2fb 2024-11-26T23:33:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664057365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/6e4f8516866142be9603dfeeb8b7f30e is 50, key is test_row_0/C:col10/1732663996656/Put/seqid=0 2024-11-26T23:33:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741857_1033 (size=12001) 2024-11-26T23:33:17,411 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/6e4f8516866142be9603dfeeb8b7f30e 2024-11-26T23:33:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/0d9c430f711e4b708e3a2f2b39187020 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/0d9c430f711e4b708e3a2f2b39187020 2024-11-26T23:33:17,440 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/0d9c430f711e4b708e3a2f2b39187020, entries=150, sequenceid=117, filesize=11.7 K 2024-11-26T23:33:17,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/06b845f8fd3a4cd9a4b2462ac1bba2fb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/06b845f8fd3a4cd9a4b2462ac1bba2fb 2024-11-26T23:33:17,460 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/06b845f8fd3a4cd9a4b2462ac1bba2fb, entries=150, sequenceid=117, filesize=11.7 K 2024-11-26T23:33:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T23:33:17,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/6e4f8516866142be9603dfeeb8b7f30e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6e4f8516866142be9603dfeeb8b7f30e 2024-11-26T23:33:17,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664057462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664057472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664057472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664057472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664057472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,484 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6e4f8516866142be9603dfeeb8b7f30e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-26T23:33:17,488 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for 56adae88c925ec88c99616a57d431fab in 190ms, sequenceid=117, compaction requested=true 2024-11-26T23:33:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T23:33:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T23:33:17,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T23:33:17,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 349 msec 2024-11-26T23:33:17,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 366 msec 2024-11-26T23:33:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:17,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T23:33:17,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f80eacab35894f94a1b09dfc4bfb1a6e is 50, key is test_row_0/A:col10/1732663997348/Put/seqid=0 2024-11-26T23:33:17,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741858_1034 (size=14391) 2024-11-26T23:33:17,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f80eacab35894f94a1b09dfc4bfb1a6e 2024-11-26T23:33:17,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664057719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664057722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664057726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664057728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664057729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/4798b740479249f9bb3f6f8a2f5dc8fa is 50, key is test_row_0/B:col10/1732663997348/Put/seqid=0 2024-11-26T23:33:17,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741859_1035 (size=12051) 2024-11-26T23:33:17,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/4798b740479249f9bb3f6f8a2f5dc8fa 2024-11-26T23:33:17,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/5554575179434d0ca02f756b493fb62a is 50, key is test_row_0/C:col10/1732663997348/Put/seqid=0 2024-11-26T23:33:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T23:33:17,772 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:17,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-26T23:33:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:33:17,775 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:17,777 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:17,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:17,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741860_1036 (size=12051) 2024-11-26T23:33:17,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/5554575179434d0ca02f756b493fb62a 2024-11-26T23:33:17,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f80eacab35894f94a1b09dfc4bfb1a6e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f80eacab35894f94a1b09dfc4bfb1a6e 2024-11-26T23:33:17,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f80eacab35894f94a1b09dfc4bfb1a6e, entries=200, sequenceid=129, filesize=14.1 K 2024-11-26T23:33:17,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/4798b740479249f9bb3f6f8a2f5dc8fa as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/4798b740479249f9bb3f6f8a2f5dc8fa 2024-11-26T23:33:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/4798b740479249f9bb3f6f8a2f5dc8fa, entries=150, sequenceid=129, filesize=11.8 K 2024-11-26T23:33:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664057833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/5554575179434d0ca02f756b493fb62a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/5554575179434d0ca02f756b493fb62a 2024-11-26T23:33:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664057842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664057842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/5554575179434d0ca02f756b493fb62a, entries=150, sequenceid=129, filesize=11.8 K 2024-11-26T23:33:17,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 56adae88c925ec88c99616a57d431fab in 173ms, sequenceid=129, compaction requested=true 2024-11-26T23:33:17,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:17,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:17,848 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:17,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:17,849 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:17,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:17,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:17,852 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:17,852 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:17,852 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:17,852 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:17,852 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:17,852 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:17,853 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/03e5d365ffff42f6831acd7082e5276e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/539a5157586544acae7d6f61392077bf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/0d9c430f711e4b708e3a2f2b39187020, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f80eacab35894f94a1b09dfc4bfb1a6e] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=49.3 K 2024-11-26T23:33:17,853 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/de7c5382df70452d8bea775ab7288eb1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d60042642a7e4c509d4a46a3602a0d92, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/06b845f8fd3a4cd9a4b2462ac1bba2fb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/4798b740479249f9bb3f6f8a2f5dc8fa] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=47.1 K 2024-11-26T23:33:17,854 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting de7c5382df70452d8bea775ab7288eb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732663994726 2024-11-26T23:33:17,854 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e5d365ffff42f6831acd7082e5276e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732663994726 2024-11-26T23:33:17,855 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d60042642a7e4c509d4a46a3602a0d92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732663995888 2024-11-26T23:33:17,855 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 539a5157586544acae7d6f61392077bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732663995888 2024-11-26T23:33:17,856 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b845f8fd3a4cd9a4b2462ac1bba2fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732663996621 2024-11-26T23:33:17,856 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d9c430f711e4b708e3a2f2b39187020, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732663996621 2024-11-26T23:33:17,856 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4798b740479249f9bb3f6f8a2f5dc8fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732663997324 2024-11-26T23:33:17,856 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80eacab35894f94a1b09dfc4bfb1a6e, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732663997324 2024-11-26T23:33:17,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:17,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:17,860 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:17,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:33:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:33:17,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6fcc21d1379649e484b952203bcaea12 is 50, key is test_row_0/A:col10/1732663997721/Put/seqid=0 2024-11-26T23:33:17,893 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#25 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:17,894 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/475687b876bb46b2bfa27ac7df4343ec is 50, key is test_row_0/A:col10/1732663997348/Put/seqid=0 2024-11-26T23:33:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664057889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664057890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:17,904 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#26 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:17,905 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/248a2574204740f2a3bd02a70a411d6f is 50, key is test_row_0/B:col10/1732663997348/Put/seqid=0 2024-11-26T23:33:17,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741861_1037 (size=12325) 2024-11-26T23:33:17,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741862_1038 (size=14541) 2024-11-26T23:33:17,929 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/475687b876bb46b2bfa27ac7df4343ec as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/475687b876bb46b2bfa27ac7df4343ec 2024-11-26T23:33:17,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:17,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:17,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:17,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:17,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:17,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:17,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741863_1039 (size=12325) 2024-11-26T23:33:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:17,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6fcc21d1379649e484b952203bcaea12 2024-11-26T23:33:17,951 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/248a2574204740f2a3bd02a70a411d6f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/248a2574204740f2a3bd02a70a411d6f 2024-11-26T23:33:17,952 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into 475687b876bb46b2bfa27ac7df4343ec(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:17,958 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:17,958 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=12, startTime=1732663997848; duration=0sec 2024-11-26T23:33:17,958 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:17,958 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:17,958 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:17,964 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:17,964 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:17,965 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:17,965 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d0770b46a436454abf95196ab3a70931, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/61e15f046c664e858726dd5fedba9c0d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6e4f8516866142be9603dfeeb8b7f30e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/5554575179434d0ca02f756b493fb62a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=47.1 K 2024-11-26T23:33:17,966 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0770b46a436454abf95196ab3a70931, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732663994726 2024-11-26T23:33:17,967 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61e15f046c664e858726dd5fedba9c0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732663995888 2024-11-26T23:33:17,967 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e4f8516866142be9603dfeeb8b7f30e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732663996621 2024-11-26T23:33:17,968 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5554575179434d0ca02f756b493fb62a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732663997324 2024-11-26T23:33:17,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e846dedf84a14e64b57f04059e22a654 is 50, key is test_row_0/B:col10/1732663997721/Put/seqid=0 2024-11-26T23:33:17,984 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into 248a2574204740f2a3bd02a70a411d6f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:17,984 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:17,984 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=12, startTime=1732663997848; duration=0sec 2024-11-26T23:33:17,984 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:17,984 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:17,995 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:17,996 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/a1f2f537463a481b9eebbab85447b393 is 50, key is test_row_0/C:col10/1732663997348/Put/seqid=0 2024-11-26T23:33:17,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741864_1040 (size=12151) 2024-11-26T23:33:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664058002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664058002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741865_1041 (size=12325) 2024-11-26T23:33:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664058042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664058051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664058051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:33:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664058212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664058212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664058351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664058362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664058362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:33:18,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e846dedf84a14e64b57f04059e22a654 2024-11-26T23:33:18,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/dc1fb09efb174a26bc03eb5fadef2682 is 50, key is test_row_0/C:col10/1732663997721/Put/seqid=0 2024-11-26T23:33:18,425 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/a1f2f537463a481b9eebbab85447b393 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a1f2f537463a481b9eebbab85447b393 2024-11-26T23:33:18,436 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into a1f2f537463a481b9eebbab85447b393(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:18,437 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:18,437 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=12, startTime=1732663997852; duration=0sec 2024-11-26T23:33:18,437 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:18,437 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:18,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741866_1042 (size=12151) 2024-11-26T23:33:18,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/dc1fb09efb174a26bc03eb5fadef2682 2024-11-26T23:33:18,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6fcc21d1379649e484b952203bcaea12 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6fcc21d1379649e484b952203bcaea12 2024-11-26T23:33:18,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6fcc21d1379649e484b952203bcaea12, entries=200, sequenceid=156, filesize=14.2 K 2024-11-26T23:33:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e846dedf84a14e64b57f04059e22a654 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e846dedf84a14e64b57f04059e22a654 2024-11-26T23:33:18,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e846dedf84a14e64b57f04059e22a654, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T23:33:18,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/dc1fb09efb174a26bc03eb5fadef2682 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/dc1fb09efb174a26bc03eb5fadef2682 2024-11-26T23:33:18,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/dc1fb09efb174a26bc03eb5fadef2682, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T23:33:18,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 56adae88c925ec88c99616a57d431fab in 634ms, sequenceid=156, compaction requested=false 2024-11-26T23:33:18,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:18,532 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:18,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T23:33:18,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f1161d9256134cb98009d0d053440097 is 50, key is test_row_0/A:col10/1732663997885/Put/seqid=0 2024-11-26T23:33:18,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:18,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741867_1043 (size=16931) 2024-11-26T23:33:18,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f1161d9256134cb98009d0d053440097 2024-11-26T23:33:18,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f04918b4a6ac4e9494605f68fac20a28 is 50, key is test_row_0/B:col10/1732663997885/Put/seqid=0 2024-11-26T23:33:18,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741868_1044 (size=12151) 2024-11-26T23:33:18,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664058613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664058617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:18,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664058722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664058731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664058861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:18,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:18,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:18,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:18,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664058872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664058872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:33:18,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664058932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664058942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f04918b4a6ac4e9494605f68fac20a28 2024-11-26T23:33:19,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/aa303a5e741d41abbcd7689b7c3c6d6d is 50, key is test_row_0/C:col10/1732663997885/Put/seqid=0 2024-11-26T23:33:19,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741869_1045 (size=12151) 2024-11-26T23:33:19,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:19,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:19,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664059241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664059252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:19,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:19,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/aa303a5e741d41abbcd7689b7c3c6d6d 2024-11-26T23:33:19,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f1161d9256134cb98009d0d053440097 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f1161d9256134cb98009d0d053440097 2024-11-26T23:33:19,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f1161d9256134cb98009d0d053440097, entries=250, sequenceid=171, filesize=16.5 K 2024-11-26T23:33:19,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f04918b4a6ac4e9494605f68fac20a28 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f04918b4a6ac4e9494605f68fac20a28 2024-11-26T23:33:19,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f04918b4a6ac4e9494605f68fac20a28, entries=150, sequenceid=171, filesize=11.9 K 2024-11-26T23:33:19,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/aa303a5e741d41abbcd7689b7c3c6d6d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aa303a5e741d41abbcd7689b7c3c6d6d 2024-11-26T23:33:19,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aa303a5e741d41abbcd7689b7c3c6d6d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-26T23:33:19,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 56adae88c925ec88c99616a57d431fab in 948ms, sequenceid=171, compaction requested=true 2024-11-26T23:33:19,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:19,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:19,481 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:19,481 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:19,484 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:19,484 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:19,484 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:19,484 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:19,484 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,484 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,484 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/475687b876bb46b2bfa27ac7df4343ec, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6fcc21d1379649e484b952203bcaea12, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f1161d9256134cb98009d0d053440097] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=42.8 K 2024-11-26T23:33:19,484 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/248a2574204740f2a3bd02a70a411d6f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e846dedf84a14e64b57f04059e22a654, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f04918b4a6ac4e9494605f68fac20a28] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=35.8 K 2024-11-26T23:33:19,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:33:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:33:19,486 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 475687b876bb46b2bfa27ac7df4343ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732663997324 2024-11-26T23:33:19,486 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 248a2574204740f2a3bd02a70a411d6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732663997324 2024-11-26T23:33:19,487 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fcc21d1379649e484b952203bcaea12, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732663997721 2024-11-26T23:33:19,487 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1161d9256134cb98009d0d053440097, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732663997883 2024-11-26T23:33:19,488 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e846dedf84a14e64b57f04059e22a654, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732663997721 2024-11-26T23:33:19,489 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f04918b4a6ac4e9494605f68fac20a28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732663997885 2024-11-26T23:33:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/4ad9064807a842db87c2c5b2065c3a55 is 50, key is test_row_0/A:col10/1732663998607/Put/seqid=0 2024-11-26T23:33:19,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:19,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:19,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:19,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:19,508 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:19,508 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/9ac152ebf36244619b0b5fcf4d64ef80 is 50, key is test_row_0/A:col10/1732663997885/Put/seqid=0 2024-11-26T23:33:19,514 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:19,515 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/b03781781cda4870a2b7f774f681bb9e is 50, key is test_row_0/B:col10/1732663997885/Put/seqid=0 2024-11-26T23:33:19,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741871_1047 (size=12151) 2024-11-26T23:33:19,551 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/4ad9064807a842db87c2c5b2065c3a55 2024-11-26T23:33:19,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741872_1048 (size=12527) 2024-11-26T23:33:19,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741870_1046 (size=12527) 2024-11-26T23:33:19,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/466e7408e6e44bf29e5bfcdee3412884 is 50, key is test_row_0/B:col10/1732663998607/Put/seqid=0 2024-11-26T23:33:19,590 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/9ac152ebf36244619b0b5fcf4d64ef80 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/9ac152ebf36244619b0b5fcf4d64ef80 2024-11-26T23:33:19,605 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into 9ac152ebf36244619b0b5fcf4d64ef80(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:19,605 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:19,605 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=13, startTime=1732663999480; duration=0sec 2024-11-26T23:33:19,605 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:19,605 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:19,605 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:19,608 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:19,608 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:19,608 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,608 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a1f2f537463a481b9eebbab85447b393, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/dc1fb09efb174a26bc03eb5fadef2682, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aa303a5e741d41abbcd7689b7c3c6d6d] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=35.8 K 2024-11-26T23:33:19,609 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f2f537463a481b9eebbab85447b393, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732663997324 2024-11-26T23:33:19,610 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc1fb09efb174a26bc03eb5fadef2682, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732663997721 2024-11-26T23:33:19,610 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa303a5e741d41abbcd7689b7c3c6d6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732663997885 2024-11-26T23:33:19,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741873_1049 (size=12151) 2024-11-26T23:33:19,613 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/466e7408e6e44bf29e5bfcdee3412884 2024-11-26T23:33:19,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/81d611c5ccc84cd6a6a63544b06e9d95 is 50, key is test_row_0/C:col10/1732663998607/Put/seqid=0 2024-11-26T23:33:19,647 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:19,647 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/6b3dbaab13c74b02b410a064fe3629ff is 50, key is test_row_0/C:col10/1732663997885/Put/seqid=0 2024-11-26T23:33:19,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741875_1051 (size=12527) 2024-11-26T23:33:19,681 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/6b3dbaab13c74b02b410a064fe3629ff as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6b3dbaab13c74b02b410a064fe3629ff 2024-11-26T23:33:19,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741874_1050 (size=12151) 2024-11-26T23:33:19,688 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/81d611c5ccc84cd6a6a63544b06e9d95 2024-11-26T23:33:19,692 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into 6b3dbaab13c74b02b410a064fe3629ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:19,692 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:19,692 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=13, startTime=1732663999496; duration=0sec 2024-11-26T23:33:19,693 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:19,693 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/4ad9064807a842db87c2c5b2065c3a55 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/4ad9064807a842db87c2c5b2065c3a55 2024-11-26T23:33:19,705 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/4ad9064807a842db87c2c5b2065c3a55, entries=150, sequenceid=195, filesize=11.9 K 2024-11-26T23:33:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/466e7408e6e44bf29e5bfcdee3412884 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/466e7408e6e44bf29e5bfcdee3412884 2024-11-26T23:33:19,722 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/466e7408e6e44bf29e5bfcdee3412884, entries=150, sequenceid=195, filesize=11.9 K 2024-11-26T23:33:19,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/81d611c5ccc84cd6a6a63544b06e9d95 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/81d611c5ccc84cd6a6a63544b06e9d95 2024-11-26T23:33:19,742 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/81d611c5ccc84cd6a6a63544b06e9d95, entries=150, sequenceid=195, filesize=11.9 K 2024-11-26T23:33:19,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 56adae88c925ec88c99616a57d431fab in 260ms, sequenceid=195, compaction requested=false 2024-11-26T23:33:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T23:33:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T23:33:19,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T23:33:19,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9710 sec 2024-11-26T23:33:19,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9800 sec 2024-11-26T23:33:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:19,777 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:19,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:19,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/36bc41183ee7407db0b1a000983be6fe is 50, key is test_row_0/A:col10/1732663999775/Put/seqid=0 2024-11-26T23:33:19,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741876_1052 (size=14541) 2024-11-26T23:33:19,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/36bc41183ee7407db0b1a000983be6fe 2024-11-26T23:33:19,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/fb598b11f46441139fe7e14f6d618bc6 is 50, key is test_row_0/B:col10/1732663999775/Put/seqid=0 2024-11-26T23:33:19,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741877_1053 (size=12151) 2024-11-26T23:33:19,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664059859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664059862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664059873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,883 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664059891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664059892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,893 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,894 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:33:19,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:19,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-26T23:33:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:33:19,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:19,931 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:19,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664059972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664059972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:19,980 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/b03781781cda4870a2b7f774f681bb9e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/b03781781cda4870a2b7f774f681bb9e 2024-11-26T23:33:19,992 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into b03781781cda4870a2b7f774f681bb9e(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:19,994 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:19,994 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=13, startTime=1732663999481; duration=0sec 2024-11-26T23:33:19,994 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:19,994 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:33:20,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T23:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664060181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:20,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664060182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:20,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/fb598b11f46441139fe7e14f6d618bc6 2024-11-26T23:33:20,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T23:33:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:33:20,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ff3d4ebeb2f74681a4c4e78be79a13b3 is 50, key is test_row_0/C:col10/1732663999775/Put/seqid=0 2024-11-26T23:33:20,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741878_1054 (size=12151) 2024-11-26T23:33:20,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T23:33:20,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:20,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664060491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664060492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:20,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T23:33:20,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:20,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:33:20,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ff3d4ebeb2f74681a4c4e78be79a13b3 2024-11-26T23:33:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/36bc41183ee7407db0b1a000983be6fe as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/36bc41183ee7407db0b1a000983be6fe 2024-11-26T23:33:20,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/36bc41183ee7407db0b1a000983be6fe, entries=200, sequenceid=208, filesize=14.2 K 2024-11-26T23:33:20,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/fb598b11f46441139fe7e14f6d618bc6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/fb598b11f46441139fe7e14f6d618bc6 2024-11-26T23:33:20,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/fb598b11f46441139fe7e14f6d618bc6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T23:33:20,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ff3d4ebeb2f74681a4c4e78be79a13b3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ff3d4ebeb2f74681a4c4e78be79a13b3 2024-11-26T23:33:20,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ff3d4ebeb2f74681a4c4e78be79a13b3, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T23:33:20,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 56adae88c925ec88c99616a57d431fab in 924ms, sequenceid=208, compaction requested=true 2024-11-26T23:33:20,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:20,702 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:20,703 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:20,704 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:20,704 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,704 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/9ac152ebf36244619b0b5fcf4d64ef80, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/4ad9064807a842db87c2c5b2065c3a55, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/36bc41183ee7407db0b1a000983be6fe] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=38.3 K 2024-11-26T23:33:20,705 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac152ebf36244619b0b5fcf4d64ef80, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732663997885 2024-11-26T23:33:20,705 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ad9064807a842db87c2c5b2065c3a55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732663998599 2024-11-26T23:33:20,706 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bc41183ee7407db0b1a000983be6fe, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732663999765 2024-11-26T23:33:20,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:20,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T23:33:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,712 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:33:20,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:20,714 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:20,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:20,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:20,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:20,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:20,716 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:20,716 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:20,716 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,716 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/b03781781cda4870a2b7f774f681bb9e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/466e7408e6e44bf29e5bfcdee3412884, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/fb598b11f46441139fe7e14f6d618bc6] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=36.0 K 2024-11-26T23:33:20,717 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b03781781cda4870a2b7f774f681bb9e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732663997885 2024-11-26T23:33:20,718 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466e7408e6e44bf29e5bfcdee3412884, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732663998599 2024-11-26T23:33:20,718 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb598b11f46441139fe7e14f6d618bc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732663999770 2024-11-26T23:33:20,724 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:20,725 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/e1f7a92a1d4e4148b8f348ae61a9202e is 50, key is test_row_0/A:col10/1732663999775/Put/seqid=0 2024-11-26T23:33:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/3bc8ff2748504e9a9f346d82f68dd239 is 50, key is test_row_0/A:col10/1732663999860/Put/seqid=0 2024-11-26T23:33:20,746 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#44 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:20,747 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c20584933a3443c89d871c34c1a54985 is 50, key is test_row_0/B:col10/1732663999775/Put/seqid=0 2024-11-26T23:33:20,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741879_1055 (size=12629) 2024-11-26T23:33:20,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741880_1056 (size=12151) 2024-11-26T23:33:20,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741881_1057 (size=12629) 2024-11-26T23:33:20,806 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c20584933a3443c89d871c34c1a54985 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c20584933a3443c89d871c34c1a54985 2024-11-26T23:33:20,819 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into c20584933a3443c89d871c34c1a54985(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:20,820 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:20,820 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=13, startTime=1732664000713; duration=0sec 2024-11-26T23:33:20,820 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:20,820 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:20,820 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:20,823 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:20,823 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:20,823 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:20,823 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6b3dbaab13c74b02b410a064fe3629ff, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/81d611c5ccc84cd6a6a63544b06e9d95, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ff3d4ebeb2f74681a4c4e78be79a13b3] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=36.0 K 2024-11-26T23:33:20,824 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3dbaab13c74b02b410a064fe3629ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732663997885 2024-11-26T23:33:20,825 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81d611c5ccc84cd6a6a63544b06e9d95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732663998599 2024-11-26T23:33:20,825 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff3d4ebeb2f74681a4c4e78be79a13b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732663999770 2024-11-26T23:33:20,837 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:20,838 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/1c1394374d9b48269ef35f6ed2d95fa6 is 50, key is test_row_0/C:col10/1732663999775/Put/seqid=0 2024-11-26T23:33:20,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741882_1058 (size=12629) 2024-11-26T23:33:20,856 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/1c1394374d9b48269ef35f6ed2d95fa6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/1c1394374d9b48269ef35f6ed2d95fa6 2024-11-26T23:33:20,866 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into 1c1394374d9b48269ef35f6ed2d95fa6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:20,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:20,866 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=13, startTime=1732664000714; duration=0sec 2024-11-26T23:33:20,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:20,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:21,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:21,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664061022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664061023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:33:21,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664061132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664061132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,157 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/e1f7a92a1d4e4148b8f348ae61a9202e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e1f7a92a1d4e4148b8f348ae61a9202e 2024-11-26T23:33:21,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/3bc8ff2748504e9a9f346d82f68dd239 2024-11-26T23:33:21,168 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into e1f7a92a1d4e4148b8f348ae61a9202e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:21,168 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:21,168 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=13, startTime=1732664000702; duration=0sec 2024-11-26T23:33:21,168 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:21,168 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e26f05b602ac4f90b62e584358846cc4 is 50, key is test_row_0/B:col10/1732663999860/Put/seqid=0 2024-11-26T23:33:21,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741883_1059 (size=12151) 2024-11-26T23:33:21,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e26f05b602ac4f90b62e584358846cc4 2024-11-26T23:33:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/8b3be3a2fe924374bd7cd29d5b43ece3 is 50, key is test_row_0/C:col10/1732663999860/Put/seqid=0 2024-11-26T23:33:21,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741884_1060 (size=12151) 2024-11-26T23:33:21,215 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/8b3be3a2fe924374bd7cd29d5b43ece3 2024-11-26T23:33:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/3bc8ff2748504e9a9f346d82f68dd239 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/3bc8ff2748504e9a9f346d82f68dd239 2024-11-26T23:33:21,235 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/3bc8ff2748504e9a9f346d82f68dd239, entries=150, sequenceid=234, filesize=11.9 K 2024-11-26T23:33:21,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e26f05b602ac4f90b62e584358846cc4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e26f05b602ac4f90b62e584358846cc4 2024-11-26T23:33:21,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e26f05b602ac4f90b62e584358846cc4, entries=150, sequenceid=234, filesize=11.9 K 2024-11-26T23:33:21,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/8b3be3a2fe924374bd7cd29d5b43ece3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/8b3be3a2fe924374bd7cd29d5b43ece3 2024-11-26T23:33:21,251 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/8b3be3a2fe924374bd7cd29d5b43ece3, entries=150, sequenceid=234, filesize=11.9 K 2024-11-26T23:33:21,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 56adae88c925ec88c99616a57d431fab in 540ms, sequenceid=234, compaction requested=false 2024-11-26T23:33:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T23:33:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T23:33:21,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T23:33:21,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3230 sec 2024-11-26T23:33:21,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3350 sec 2024-11-26T23:33:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:21,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:21,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T23:33:21,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/5a255c5660da41ddb47eb9bfd932f15c is 50, key is test_row_0/A:col10/1732664001020/Put/seqid=0 2024-11-26T23:33:21,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741885_1061 (size=16931) 2024-11-26T23:33:21,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/5a255c5660da41ddb47eb9bfd932f15c 2024-11-26T23:33:21,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/88613be832cd4bd68829c1407fa75f7b is 50, key is test_row_0/B:col10/1732664001020/Put/seqid=0 2024-11-26T23:33:21,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741886_1062 (size=12151) 2024-11-26T23:33:21,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664061403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664061405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664061512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664061521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664061722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664061731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/88613be832cd4bd68829c1407fa75f7b 2024-11-26T23:33:21,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d15d8193e1ef42e8bc2844c8ab21c7b7 is 50, key is test_row_0/C:col10/1732664001020/Put/seqid=0 2024-11-26T23:33:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741887_1063 (size=12151) 2024-11-26T23:33:21,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664061902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,905 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664061921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664061921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:21,925 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:21,926 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664062031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664062041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:33:22,081 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:22,082 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-26T23:33:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T23:33:22,085 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:22,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:22,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T23:33:22,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d15d8193e1ef42e8bc2844c8ab21c7b7 2024-11-26T23:33:22,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T23:33:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:22,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/5a255c5660da41ddb47eb9bfd932f15c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/5a255c5660da41ddb47eb9bfd932f15c 2024-11-26T23:33:22,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/5a255c5660da41ddb47eb9bfd932f15c, entries=250, sequenceid=250, filesize=16.5 K 2024-11-26T23:33:22,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/88613be832cd4bd68829c1407fa75f7b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/88613be832cd4bd68829c1407fa75f7b 2024-11-26T23:33:22,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/88613be832cd4bd68829c1407fa75f7b, entries=150, sequenceid=250, filesize=11.9 K 2024-11-26T23:33:22,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d15d8193e1ef42e8bc2844c8ab21c7b7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d15d8193e1ef42e8bc2844c8ab21c7b7 2024-11-26T23:33:22,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d15d8193e1ef42e8bc2844c8ab21c7b7, entries=150, sequenceid=250, filesize=11.9 K 2024-11-26T23:33:22,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 56adae88c925ec88c99616a57d431fab in 925ms, sequenceid=250, compaction requested=true 2024-11-26T23:33:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:22,270 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:22,270 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:22,272 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:22,272 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:22,272 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:22,272 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,272 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:22,273 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,273 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c20584933a3443c89d871c34c1a54985, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e26f05b602ac4f90b62e584358846cc4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/88613be832cd4bd68829c1407fa75f7b] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=36.1 K 2024-11-26T23:33:22,273 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e1f7a92a1d4e4148b8f348ae61a9202e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/3bc8ff2748504e9a9f346d82f68dd239, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/5a255c5660da41ddb47eb9bfd932f15c] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=40.7 K 2024-11-26T23:33:22,273 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c20584933a3443c89d871c34c1a54985, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732663999770 2024-11-26T23:33:22,273 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f7a92a1d4e4148b8f348ae61a9202e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732663999770 2024-11-26T23:33:22,274 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e26f05b602ac4f90b62e584358846cc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732663999841 2024-11-26T23:33:22,274 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc8ff2748504e9a9f346d82f68dd239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732663999841 2024-11-26T23:33:22,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:22,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:22,274 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a255c5660da41ddb47eb9bfd932f15c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732664001008 2024-11-26T23:33:22,274 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 88613be832cd4bd68829c1407fa75f7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732664001020 2024-11-26T23:33:22,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:22,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:22,286 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:22,287 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:22,287 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/dab6ce1062874ed8a9d7bd7a10ef975e is 50, key is test_row_0/B:col10/1732664001020/Put/seqid=0 2024-11-26T23:33:22,289 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f83a3efff8a2457fa7a9cc763ab90591 is 50, key is test_row_0/A:col10/1732664001020/Put/seqid=0 2024-11-26T23:33:22,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741889_1065 (size=12731) 2024-11-26T23:33:22,311 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/f83a3efff8a2457fa7a9cc763ab90591 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f83a3efff8a2457fa7a9cc763ab90591 2024-11-26T23:33:22,320 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into f83a3efff8a2457fa7a9cc763ab90591(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:22,320 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:22,320 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=13, startTime=1732664002270; duration=0sec 2024-11-26T23:33:22,320 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:22,320 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:22,320 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:22,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741888_1064 (size=12731) 2024-11-26T23:33:22,324 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:22,325 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:22,325 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,325 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/1c1394374d9b48269ef35f6ed2d95fa6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/8b3be3a2fe924374bd7cd29d5b43ece3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d15d8193e1ef42e8bc2844c8ab21c7b7] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=36.1 K 2024-11-26T23:33:22,326 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c1394374d9b48269ef35f6ed2d95fa6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732663999770 2024-11-26T23:33:22,327 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b3be3a2fe924374bd7cd29d5b43ece3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732663999841 2024-11-26T23:33:22,328 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d15d8193e1ef42e8bc2844c8ab21c7b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732664001020 2024-11-26T23:33:22,330 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/dab6ce1062874ed8a9d7bd7a10ef975e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dab6ce1062874ed8a9d7bd7a10ef975e 2024-11-26T23:33:22,343 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into dab6ce1062874ed8a9d7bd7a10ef975e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:22,343 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:22,343 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=13, startTime=1732664002270; duration=0sec 2024-11-26T23:33:22,344 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:22,344 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:22,357 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:22,358 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/f17f16c584b743a9aa9638f4e0538867 is 50, key is test_row_0/C:col10/1732664001020/Put/seqid=0 2024-11-26T23:33:22,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741890_1066 (size=12731) 2024-11-26T23:33:22,396 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/f17f16c584b743a9aa9638f4e0538867 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f17f16c584b743a9aa9638f4e0538867 2024-11-26T23:33:22,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T23:33:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,400 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T23:33:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T23:33:22,407 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into f17f16c584b743a9aa9638f4e0538867(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:22,407 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:22,407 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=13, startTime=1732664002275; duration=0sec 2024-11-26T23:33:22,407 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:22,407 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/e7bda0cbd0474ac68b7a303683f16e8e is 50, key is test_row_0/A:col10/1732664001403/Put/seqid=0 2024-11-26T23:33:22,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741891_1067 (size=12301) 2024-11-26T23:33:22,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/e7bda0cbd0474ac68b7a303683f16e8e 2024-11-26T23:33:22,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/404fe875109f45738b2e48be63ffb168 is 50, key is test_row_0/B:col10/1732664001403/Put/seqid=0 2024-11-26T23:33:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741892_1068 (size=12301) 2024-11-26T23:33:22,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/404fe875109f45738b2e48be63ffb168 2024-11-26T23:33:22,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/88878b403da4420b808a0e32f7762404 is 50, key is test_row_0/C:col10/1732664001403/Put/seqid=0 2024-11-26T23:33:22,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741893_1069 (size=12301) 2024-11-26T23:33:22,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/88878b403da4420b808a0e32f7762404 2024-11-26T23:33:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/e7bda0cbd0474ac68b7a303683f16e8e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e7bda0cbd0474ac68b7a303683f16e8e 2024-11-26T23:33:22,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e7bda0cbd0474ac68b7a303683f16e8e, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T23:33:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/404fe875109f45738b2e48be63ffb168 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/404fe875109f45738b2e48be63ffb168 2024-11-26T23:33:22,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/404fe875109f45738b2e48be63ffb168, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T23:33:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/88878b403da4420b808a0e32f7762404 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/88878b403da4420b808a0e32f7762404 2024-11-26T23:33:22,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/88878b403da4420b808a0e32f7762404, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T23:33:22,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 56adae88c925ec88c99616a57d431fab in 129ms, sequenceid=275, compaction requested=false 2024-11-26T23:33:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-26T23:33:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-26T23:33:22,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T23:33:22,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 445 msec 2024-11-26T23:33:22,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 456 msec 2024-11-26T23:33:22,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:22,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:22,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8570353ff780474792fefde0afc3f3bb is 50, key is test_row_0/A:col10/1732664002567/Put/seqid=0 2024-11-26T23:33:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741894_1070 (size=9857) 2024-11-26T23:33:22,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8570353ff780474792fefde0afc3f3bb 2024-11-26T23:33:22,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/9e7dfbdb2d8c4927a2fd33b8070d232b is 50, key is test_row_0/B:col10/1732664002567/Put/seqid=0 2024-11-26T23:33:22,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741895_1071 (size=9857) 2024-11-26T23:33:22,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/9e7dfbdb2d8c4927a2fd33b8070d232b 2024-11-26T23:33:22,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664062643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664062644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/76ef7923405e4ff0a8e74f650cbd44c2 is 50, key is test_row_0/C:col10/1732664002567/Put/seqid=0 2024-11-26T23:33:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741896_1072 (size=9857) 2024-11-26T23:33:22,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/76ef7923405e4ff0a8e74f650cbd44c2 2024-11-26T23:33:22,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8570353ff780474792fefde0afc3f3bb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8570353ff780474792fefde0afc3f3bb 2024-11-26T23:33:22,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8570353ff780474792fefde0afc3f3bb, entries=100, sequenceid=287, filesize=9.6 K 2024-11-26T23:33:22,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/9e7dfbdb2d8c4927a2fd33b8070d232b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9e7dfbdb2d8c4927a2fd33b8070d232b 2024-11-26T23:33:22,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9e7dfbdb2d8c4927a2fd33b8070d232b, entries=100, sequenceid=287, filesize=9.6 K 2024-11-26T23:33:22,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/76ef7923405e4ff0a8e74f650cbd44c2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/76ef7923405e4ff0a8e74f650cbd44c2 2024-11-26T23:33:22,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/76ef7923405e4ff0a8e74f650cbd44c2, entries=100, sequenceid=287, filesize=9.6 K 2024-11-26T23:33:22,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 56adae88c925ec88c99616a57d431fab in 135ms, sequenceid=287, compaction requested=true 2024-11-26T23:33:22,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:22,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:22,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:22,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:22,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:22,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:22,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T23:33:22,705 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:22,705 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:22,706 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:22,706 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:22,706 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,707 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dab6ce1062874ed8a9d7bd7a10ef975e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/404fe875109f45738b2e48be63ffb168, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9e7dfbdb2d8c4927a2fd33b8070d232b] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=34.1 K 2024-11-26T23:33:22,707 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:22,707 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:22,707 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,707 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f83a3efff8a2457fa7a9cc763ab90591, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e7bda0cbd0474ac68b7a303683f16e8e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8570353ff780474792fefde0afc3f3bb] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=34.1 K 2024-11-26T23:33:22,708 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting dab6ce1062874ed8a9d7bd7a10ef975e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732664001020 2024-11-26T23:33:22,708 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83a3efff8a2457fa7a9cc763ab90591, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732664001020 2024-11-26T23:33:22,708 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 404fe875109f45738b2e48be63ffb168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732664001369 2024-11-26T23:33:22,708 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7bda0cbd0474ac68b7a303683f16e8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732664001369 2024-11-26T23:33:22,709 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7dfbdb2d8c4927a2fd33b8070d232b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732664002563 2024-11-26T23:33:22,710 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8570353ff780474792fefde0afc3f3bb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732664002563 2024-11-26T23:33:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T23:33:22,712 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:22,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-26T23:33:22,715 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T23:33:22,716 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:22,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:22,728 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:22,728 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:22,729 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/e4df861bb3cd44f38ff61f842e265274 is 50, key is test_row_0/A:col10/1732664002567/Put/seqid=0 2024-11-26T23:33:22,730 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c6cdfaace84a452583216bed7a51b9fb is 50, key is test_row_0/B:col10/1732664002567/Put/seqid=0 2024-11-26T23:33:22,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741897_1073 (size=12983) 2024-11-26T23:33:22,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741898_1074 (size=12983) 2024-11-26T23:33:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:22,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:22,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:33:22,757 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c6cdfaace84a452583216bed7a51b9fb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c6cdfaace84a452583216bed7a51b9fb 2024-11-26T23:33:22,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/efeb3ffe89344e259aebe14f37a681f3 is 50, key is test_row_0/A:col10/1732664002642/Put/seqid=0 2024-11-26T23:33:22,769 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into c6cdfaace84a452583216bed7a51b9fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:22,769 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:22,769 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=13, startTime=1732664002705; duration=0sec 2024-11-26T23:33:22,769 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:22,769 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:22,770 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:22,771 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:22,771 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:22,772 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,772 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f17f16c584b743a9aa9638f4e0538867, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/88878b403da4420b808a0e32f7762404, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/76ef7923405e4ff0a8e74f650cbd44c2] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=34.1 K 2024-11-26T23:33:22,772 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f17f16c584b743a9aa9638f4e0538867, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732664001020 2024-11-26T23:33:22,773 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 88878b403da4420b808a0e32f7762404, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732664001369 2024-11-26T23:33:22,774 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 76ef7923405e4ff0a8e74f650cbd44c2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732664002563 2024-11-26T23:33:22,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664062774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664062776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741899_1075 (size=17181) 2024-11-26T23:33:22,793 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:22,794 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/3afa672130d74188bc894239842a5a0f is 50, key is test_row_0/C:col10/1732664002567/Put/seqid=0 2024-11-26T23:33:22,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741900_1076 (size=12983) 2024-11-26T23:33:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T23:33:22,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T23:33:22,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:22,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:22,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664062882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664062882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T23:33:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:23,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T23:33:23,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664063092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664063092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,149 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/e4df861bb3cd44f38ff61f842e265274 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e4df861bb3cd44f38ff61f842e265274 2024-11-26T23:33:23,157 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into e4df861bb3cd44f38ff61f842e265274(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:23,157 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:23,157 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=13, startTime=1732664002704; duration=0sec 2024-11-26T23:33:23,157 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:23,157 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:23,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T23:33:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:23,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:23,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/efeb3ffe89344e259aebe14f37a681f3 2024-11-26T23:33:23,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/76780be895364a939fc26b3e269784aa is 50, key is test_row_0/B:col10/1732664002642/Put/seqid=0 2024-11-26T23:33:23,227 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/3afa672130d74188bc894239842a5a0f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/3afa672130d74188bc894239842a5a0f 2024-11-26T23:33:23,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741901_1077 (size=12301) 2024-11-26T23:33:23,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/76780be895364a939fc26b3e269784aa 2024-11-26T23:33:23,237 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into 3afa672130d74188bc894239842a5a0f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:23,237 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:23,237 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=13, startTime=1732664002705; duration=0sec 2024-11-26T23:33:23,238 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:23,238 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:23,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/46157877a4e640f7b76ea3cbf57e4a96 is 50, key is test_row_0/C:col10/1732664002642/Put/seqid=0 2024-11-26T23:33:23,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741902_1078 (size=12301) 2024-11-26T23:33:23,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/46157877a4e640f7b76ea3cbf57e4a96 2024-11-26T23:33:23,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/efeb3ffe89344e259aebe14f37a681f3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/efeb3ffe89344e259aebe14f37a681f3 2024-11-26T23:33:23,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/efeb3ffe89344e259aebe14f37a681f3, entries=250, sequenceid=315, filesize=16.8 K 2024-11-26T23:33:23,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/76780be895364a939fc26b3e269784aa as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/76780be895364a939fc26b3e269784aa 2024-11-26T23:33:23,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/76780be895364a939fc26b3e269784aa, entries=150, sequenceid=315, filesize=12.0 K 2024-11-26T23:33:23,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/46157877a4e640f7b76ea3cbf57e4a96 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/46157877a4e640f7b76ea3cbf57e4a96 2024-11-26T23:33:23,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/46157877a4e640f7b76ea3cbf57e4a96, entries=150, sequenceid=315, filesize=12.0 K 2024-11-26T23:33:23,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 56adae88c925ec88c99616a57d431fab in 555ms, sequenceid=315, compaction requested=false 2024-11-26T23:33:23,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:23,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T23:33:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:23,334 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-26T23:33:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/ccbbc686e38849b2b2c031d7edd9202c is 50, key is test_row_0/A:col10/1732664002769/Put/seqid=0 2024-11-26T23:33:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T23:33:23,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741903_1079 (size=12301) 2024-11-26T23:33:23,349 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/ccbbc686e38849b2b2c031d7edd9202c 2024-11-26T23:33:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/2edc56f621b64097805dcee4bf739d7e is 50, key is test_row_0/B:col10/1732664002769/Put/seqid=0 2024-11-26T23:33:23,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741904_1080 (size=12301) 2024-11-26T23:33:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:23,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664063471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664063473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664063581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664063581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/2edc56f621b64097805dcee4bf739d7e 2024-11-26T23:33:23,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664063792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664063792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:23,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ca3ceb89ce06491690986b2570389977 is 50, key is test_row_0/C:col10/1732664002769/Put/seqid=0 2024-11-26T23:33:23,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741905_1081 (size=12301) 2024-11-26T23:33:23,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ca3ceb89ce06491690986b2570389977 2024-11-26T23:33:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/ccbbc686e38849b2b2c031d7edd9202c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ccbbc686e38849b2b2c031d7edd9202c 2024-11-26T23:33:23,838 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ccbbc686e38849b2b2c031d7edd9202c, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T23:33:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/2edc56f621b64097805dcee4bf739d7e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2edc56f621b64097805dcee4bf739d7e 2024-11-26T23:33:23,845 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2edc56f621b64097805dcee4bf739d7e, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T23:33:23,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ca3ceb89ce06491690986b2570389977 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ca3ceb89ce06491690986b2570389977 2024-11-26T23:33:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T23:33:23,855 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ca3ceb89ce06491690986b2570389977, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T23:33:23,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 56adae88c925ec88c99616a57d431fab in 522ms, sequenceid=326, compaction requested=true 2024-11-26T23:33:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-26T23:33:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-26T23:33:23,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T23:33:23,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1410 sec 2024-11-26T23:33:23,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1490 sec 2024-11-26T23:33:24,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:24,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-26T23:33:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664064114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664064120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/bb790fd98d8a403fa6acfbd2803648d2 is 50, key is test_row_0/A:col10/1732664004105/Put/seqid=0 2024-11-26T23:33:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741906_1082 (size=17181) 2024-11-26T23:33:24,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/bb790fd98d8a403fa6acfbd2803648d2 2024-11-26T23:33:24,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e51020dc501446e092a89a1298cab761 is 50, key is test_row_0/B:col10/1732664004105/Put/seqid=0 2024-11-26T23:33:24,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741907_1083 (size=12301) 2024-11-26T23:33:24,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e51020dc501446e092a89a1298cab761 2024-11-26T23:33:24,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d3dd391d5b1245e38232ca9c2bb41e14 is 50, key is test_row_0/C:col10/1732664004105/Put/seqid=0 2024-11-26T23:33:24,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741908_1084 (size=12301) 2024-11-26T23:33:24,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d3dd391d5b1245e38232ca9c2bb41e14 2024-11-26T23:33:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/bb790fd98d8a403fa6acfbd2803648d2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/bb790fd98d8a403fa6acfbd2803648d2 2024-11-26T23:33:24,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/bb790fd98d8a403fa6acfbd2803648d2, entries=250, sequenceid=356, filesize=16.8 K 2024-11-26T23:33:24,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/e51020dc501446e092a89a1298cab761 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e51020dc501446e092a89a1298cab761 2024-11-26T23:33:24,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e51020dc501446e092a89a1298cab761, entries=150, sequenceid=356, filesize=12.0 K 2024-11-26T23:33:24,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d3dd391d5b1245e38232ca9c2bb41e14 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d3dd391d5b1245e38232ca9c2bb41e14 2024-11-26T23:33:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664064232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664064232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d3dd391d5b1245e38232ca9c2bb41e14, entries=150, sequenceid=356, filesize=12.0 K 2024-11-26T23:33:24,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 56adae88c925ec88c99616a57d431fab in 138ms, sequenceid=356, compaction requested=true 2024-11-26T23:33:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:24,245 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:24,245 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:24,247 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:24,247 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:24,247 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:24,247 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:24,247 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e4df861bb3cd44f38ff61f842e265274, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/efeb3ffe89344e259aebe14f37a681f3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ccbbc686e38849b2b2c031d7edd9202c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/bb790fd98d8a403fa6acfbd2803648d2] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=58.2 K 2024-11-26T23:33:24,247 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:24,247 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:24,247 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c6cdfaace84a452583216bed7a51b9fb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/76780be895364a939fc26b3e269784aa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2edc56f621b64097805dcee4bf739d7e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e51020dc501446e092a89a1298cab761] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=48.7 K 2024-11-26T23:33:24,248 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4df861bb3cd44f38ff61f842e265274, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732664001397 2024-11-26T23:33:24,248 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c6cdfaace84a452583216bed7a51b9fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732664001397 2024-11-26T23:33:24,248 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting efeb3ffe89344e259aebe14f37a681f3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732664002631 2024-11-26T23:33:24,248 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 76780be895364a939fc26b3e269784aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732664002631 2024-11-26T23:33:24,249 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccbbc686e38849b2b2c031d7edd9202c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732664002762 2024-11-26T23:33:24,249 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 2edc56f621b64097805dcee4bf739d7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732664002762 2024-11-26T23:33:24,249 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb790fd98d8a403fa6acfbd2803648d2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732664003467 2024-11-26T23:33:24,251 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e51020dc501446e092a89a1298cab761, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732664003467 2024-11-26T23:33:24,265 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#72 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:24,266 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/470919b9399640e2ba8990e1851fab85 is 50, key is test_row_0/A:col10/1732664004105/Put/seqid=0 2024-11-26T23:33:24,271 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:24,272 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d75c53cae7874f139224aaa9b06d4a08 is 50, key is test_row_0/B:col10/1732664004105/Put/seqid=0 2024-11-26T23:33:24,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741909_1085 (size=13119) 2024-11-26T23:33:24,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741910_1086 (size=13119) 2024-11-26T23:33:24,304 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/470919b9399640e2ba8990e1851fab85 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/470919b9399640e2ba8990e1851fab85 2024-11-26T23:33:24,313 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into 470919b9399640e2ba8990e1851fab85(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:24,313 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:24,313 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=12, startTime=1732664004245; duration=0sec 2024-11-26T23:33:24,313 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:24,313 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:24,313 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:24,316 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:24,316 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:24,316 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:24,316 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/3afa672130d74188bc894239842a5a0f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/46157877a4e640f7b76ea3cbf57e4a96, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ca3ceb89ce06491690986b2570389977, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d3dd391d5b1245e38232ca9c2bb41e14] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=48.7 K 2024-11-26T23:33:24,317 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3afa672130d74188bc894239842a5a0f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732664001397 2024-11-26T23:33:24,318 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46157877a4e640f7b76ea3cbf57e4a96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732664002631 2024-11-26T23:33:24,319 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca3ceb89ce06491690986b2570389977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732664002762 2024-11-26T23:33:24,319 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3dd391d5b1245e38232ca9c2bb41e14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732664003467 2024-11-26T23:33:24,332 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#74 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:24,333 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/80f3286126de4bbf975606c6ff1433a9 is 50, key is test_row_0/C:col10/1732664004105/Put/seqid=0 2024-11-26T23:33:24,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741911_1087 (size=13119) 2024-11-26T23:33:24,452 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:24,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:24,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/ecb72eb43cad4747a64d89c99fe03b71 is 50, key is test_row_0/A:col10/1732664004111/Put/seqid=0 2024-11-26T23:33:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741912_1088 (size=12301) 2024-11-26T23:33:24,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/ecb72eb43cad4747a64d89c99fe03b71 2024-11-26T23:33:24,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f3f2bd0d11994c519ac7fd0fc2dc6917 is 50, key is test_row_0/B:col10/1732664004111/Put/seqid=0 2024-11-26T23:33:24,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741913_1089 (size=12301) 2024-11-26T23:33:24,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f3f2bd0d11994c519ac7fd0fc2dc6917 2024-11-26T23:33:24,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/782bcd6a2d5f494ba52ea984150150a0 is 50, key is test_row_0/C:col10/1732664004111/Put/seqid=0 2024-11-26T23:33:24,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741914_1090 (size=12301) 2024-11-26T23:33:24,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664064530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664064530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664064641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664064641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,700 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d75c53cae7874f139224aaa9b06d4a08 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d75c53cae7874f139224aaa9b06d4a08 2024-11-26T23:33:24,707 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into d75c53cae7874f139224aaa9b06d4a08(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:24,707 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:24,707 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=12, startTime=1732664004245; duration=0sec 2024-11-26T23:33:24,707 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:24,707 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:24,755 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/80f3286126de4bbf975606c6ff1433a9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/80f3286126de4bbf975606c6ff1433a9 2024-11-26T23:33:24,763 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into 80f3286126de4bbf975606c6ff1433a9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:24,763 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:24,763 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=12, startTime=1732664004245; duration=0sec 2024-11-26T23:33:24,763 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:24,763 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664064851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664064851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T23:33:24,862 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:24,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-26T23:33:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T23:33:24,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:24,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:24,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:24,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/782bcd6a2d5f494ba52ea984150150a0 2024-11-26T23:33:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/ecb72eb43cad4747a64d89c99fe03b71 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ecb72eb43cad4747a64d89c99fe03b71 2024-11-26T23:33:24,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ecb72eb43cad4747a64d89c99fe03b71, entries=150, sequenceid=368, filesize=12.0 K 2024-11-26T23:33:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f3f2bd0d11994c519ac7fd0fc2dc6917 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f3f2bd0d11994c519ac7fd0fc2dc6917 2024-11-26T23:33:24,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f3f2bd0d11994c519ac7fd0fc2dc6917, entries=150, sequenceid=368, filesize=12.0 K 2024-11-26T23:33:24,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/782bcd6a2d5f494ba52ea984150150a0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/782bcd6a2d5f494ba52ea984150150a0 2024-11-26T23:33:24,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/782bcd6a2d5f494ba52ea984150150a0, entries=150, sequenceid=368, filesize=12.0 K 2024-11-26T23:33:24,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 56adae88c925ec88c99616a57d431fab in 492ms, sequenceid=368, compaction requested=false 2024-11-26T23:33:24,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T23:33:25,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T23:33:25,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,020 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:33:25,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6642e82c756f40e5be1392262be8bd03 is 50, key is test_row_0/A:col10/1732664004515/Put/seqid=0 2024-11-26T23:33:25,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741915_1091 (size=12301) 2024-11-26T23:33:25,033 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6642e82c756f40e5be1392262be8bd03 2024-11-26T23:33:25,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a353a907041040cb9824bba301c2f36c is 50, key is test_row_0/B:col10/1732664004515/Put/seqid=0 2024-11-26T23:33:25,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741916_1092 (size=12301) 2024-11-26T23:33:25,052 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a353a907041040cb9824bba301c2f36c 2024-11-26T23:33:25,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/60805fea63c347d0b1e1a022ed464607 is 50, key is test_row_0/C:col10/1732664004515/Put/seqid=0 2024-11-26T23:33:25,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741917_1093 (size=12301) 2024-11-26T23:33:25,073 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/60805fea63c347d0b1e1a022ed464607 2024-11-26T23:33:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6642e82c756f40e5be1392262be8bd03 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6642e82c756f40e5be1392262be8bd03 2024-11-26T23:33:25,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6642e82c756f40e5be1392262be8bd03, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T23:33:25,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a353a907041040cb9824bba301c2f36c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a353a907041040cb9824bba301c2f36c 2024-11-26T23:33:25,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a353a907041040cb9824bba301c2f36c, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T23:33:25,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/60805fea63c347d0b1e1a022ed464607 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/60805fea63c347d0b1e1a022ed464607 2024-11-26T23:33:25,100 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/60805fea63c347d0b1e1a022ed464607, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T23:33:25,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 56adae88c925ec88c99616a57d431fab in 80ms, sequenceid=395, compaction requested=true 2024-11-26T23:33:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-26T23:33:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-26T23:33:25,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-26T23:33:25,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 238 msec 2024-11-26T23:33:25,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 245 msec 2024-11-26T23:33:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T23:33:25,182 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:25,182 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:25,183 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:25,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-26T23:33:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T23:33:25,186 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:25,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:25,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:25,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6021c605195a4a55a356cd3feb98b053 is 50, key is test_row_0/A:col10/1732664005179/Put/seqid=0 2024-11-26T23:33:25,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741918_1094 (size=12301) 2024-11-26T23:33:25,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664065247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664065248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T23:33:25,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664065362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664065362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T23:33:25,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664065572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664065572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6021c605195a4a55a356cd3feb98b053 2024-11-26T23:33:25,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/503c1dee3b5b4372a26db6ce1b772f93 is 50, key is test_row_0/B:col10/1732664005179/Put/seqid=0 2024-11-26T23:33:25,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741919_1095 (size=12301) 2024-11-26T23:33:25,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:25,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T23:33:25,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664065881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664065881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56440 deadline: 1732664065912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,915 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732664065941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,946 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:25,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:25,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:25,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1732664065971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:25,975 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 56adae88c925ec88c99616a57d431fab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:26,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/503c1dee3b5b4372a26db6ce1b772f93 2024-11-26T23:33:26,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/90259e47496649bbb8564ef21aa5350d is 50, key is test_row_0/C:col10/1732664005179/Put/seqid=0 2024-11-26T23:33:26,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741920_1096 (size=12301) 2024-11-26T23:33:26,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:26,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T23:33:26,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664066392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:26,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664066392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:26,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:26,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/90259e47496649bbb8564ef21aa5350d 2024-11-26T23:33:26,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/6021c605195a4a55a356cd3feb98b053 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6021c605195a4a55a356cd3feb98b053 2024-11-26T23:33:26,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6021c605195a4a55a356cd3feb98b053, entries=150, sequenceid=407, filesize=12.0 K 2024-11-26T23:33:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/503c1dee3b5b4372a26db6ce1b772f93 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/503c1dee3b5b4372a26db6ce1b772f93 2024-11-26T23:33:26,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/503c1dee3b5b4372a26db6ce1b772f93, entries=150, sequenceid=407, filesize=12.0 K 2024-11-26T23:33:26,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/90259e47496649bbb8564ef21aa5350d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/90259e47496649bbb8564ef21aa5350d 2024-11-26T23:33:26,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/90259e47496649bbb8564ef21aa5350d, entries=150, sequenceid=407, filesize=12.0 K 2024-11-26T23:33:26,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 56adae88c925ec88c99616a57d431fab in 1288ms, sequenceid=407, compaction requested=true 2024-11-26T23:33:26,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:26,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:26,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:26,472 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:26,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:26,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:26,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:26,472 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:26,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:26,473 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:26,473 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:26,474 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:26,474 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:26,474 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,474 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,474 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/470919b9399640e2ba8990e1851fab85, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ecb72eb43cad4747a64d89c99fe03b71, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6642e82c756f40e5be1392262be8bd03, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6021c605195a4a55a356cd3feb98b053] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=48.8 K 2024-11-26T23:33:26,474 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d75c53cae7874f139224aaa9b06d4a08, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f3f2bd0d11994c519ac7fd0fc2dc6917, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a353a907041040cb9824bba301c2f36c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/503c1dee3b5b4372a26db6ce1b772f93] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=48.8 K 2024-11-26T23:33:26,474 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d75c53cae7874f139224aaa9b06d4a08, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732664003467 2024-11-26T23:33:26,474 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 470919b9399640e2ba8990e1851fab85, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732664003467 2024-11-26T23:33:26,475 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb72eb43cad4747a64d89c99fe03b71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732664004111 2024-11-26T23:33:26,475 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f2bd0d11994c519ac7fd0fc2dc6917, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732664004111 2024-11-26T23:33:26,475 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6642e82c756f40e5be1392262be8bd03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732664004515 2024-11-26T23:33:26,475 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting a353a907041040cb9824bba301c2f36c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732664004515 2024-11-26T23:33:26,476 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6021c605195a4a55a356cd3feb98b053, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732664005179 2024-11-26T23:33:26,476 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 503c1dee3b5b4372a26db6ce1b772f93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732664005179 2024-11-26T23:33:26,487 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:26,487 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:26,487 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/d6926de84212446abef54e3177462450 is 50, key is test_row_0/A:col10/1732664005179/Put/seqid=0 2024-11-26T23:33:26,488 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/9d3786be2ac042149ca05019eaba586a is 50, key is test_row_0/B:col10/1732664005179/Put/seqid=0 2024-11-26T23:33:26,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741922_1098 (size=13255) 2024-11-26T23:33:26,509 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/d6926de84212446abef54e3177462450 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/d6926de84212446abef54e3177462450 2024-11-26T23:33:26,515 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into d6926de84212446abef54e3177462450(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:26,516 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:26,516 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=12, startTime=1732664006472; duration=0sec 2024-11-26T23:33:26,516 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:26,516 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:26,516 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:26,517 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:26,518 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:26,518 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,518 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/80f3286126de4bbf975606c6ff1433a9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/782bcd6a2d5f494ba52ea984150150a0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/60805fea63c347d0b1e1a022ed464607, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/90259e47496649bbb8564ef21aa5350d] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=48.8 K 2024-11-26T23:33:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741921_1097 (size=13255) 2024-11-26T23:33:26,519 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f3286126de4bbf975606c6ff1433a9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732664003467 2024-11-26T23:33:26,519 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782bcd6a2d5f494ba52ea984150150a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732664004111 2024-11-26T23:33:26,520 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60805fea63c347d0b1e1a022ed464607, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732664004515 2024-11-26T23:33:26,520 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90259e47496649bbb8564ef21aa5350d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732664005179 2024-11-26T23:33:26,534 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:26,534 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/eed6c5b570d64f739780cd532f19b02e is 50, key is test_row_0/C:col10/1732664005179/Put/seqid=0 2024-11-26T23:33:26,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741923_1099 (size=13255) 2024-11-26T23:33:26,550 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/eed6c5b570d64f739780cd532f19b02e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/eed6c5b570d64f739780cd532f19b02e 2024-11-26T23:33:26,558 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into eed6c5b570d64f739780cd532f19b02e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:26,558 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:26,558 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=12, startTime=1732664006472; duration=0sec 2024-11-26T23:33:26,558 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:26,558 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:26,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T23:33:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:26,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:33:26,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/fac7b5740db541268b4034c8e76c57de is 50, key is test_row_0/A:col10/1732664005244/Put/seqid=0 2024-11-26T23:33:26,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741924_1100 (size=12301) 2024-11-26T23:33:26,926 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/9d3786be2ac042149ca05019eaba586a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9d3786be2ac042149ca05019eaba586a 2024-11-26T23:33:26,932 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into 9d3786be2ac042149ca05019eaba586a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:26,933 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:26,933 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=12, startTime=1732664006472; duration=0sec 2024-11-26T23:33:26,933 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:26,933 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:27,003 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/fac7b5740db541268b4034c8e76c57de 2024-11-26T23:33:27,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a15ac53103624d64adfa4d9089bb947d is 50, key is test_row_0/B:col10/1732664005244/Put/seqid=0 2024-11-26T23:33:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741925_1101 (size=12301) 2024-11-26T23:33:27,020 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a15ac53103624d64adfa4d9089bb947d 2024-11-26T23:33:27,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/44bfef45fee6496bba8c7b387377b51d is 50, key is test_row_0/C:col10/1732664005244/Put/seqid=0 2024-11-26T23:33:27,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741926_1102 (size=12301) 2024-11-26T23:33:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T23:33:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:27,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664067420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664067424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,448 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/44bfef45fee6496bba8c7b387377b51d 2024-11-26T23:33:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/fac7b5740db541268b4034c8e76c57de as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/fac7b5740db541268b4034c8e76c57de 2024-11-26T23:33:27,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/fac7b5740db541268b4034c8e76c57de, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T23:33:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a15ac53103624d64adfa4d9089bb947d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a15ac53103624d64adfa4d9089bb947d 2024-11-26T23:33:27,466 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a15ac53103624d64adfa4d9089bb947d, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T23:33:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/44bfef45fee6496bba8c7b387377b51d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/44bfef45fee6496bba8c7b387377b51d 2024-11-26T23:33:27,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/44bfef45fee6496bba8c7b387377b51d, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T23:33:27,474 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 56adae88c925ec88c99616a57d431fab in 892ms, sequenceid=433, compaction requested=false 2024-11-26T23:33:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-26T23:33:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-26T23:33:27,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T23:33:27,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2910 sec 2024-11-26T23:33:27,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2990 sec 2024-11-26T23:33:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:27,536 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:27,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-26T23:33:27,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/535a25c6e617437ba590f33f58c2ecd1 is 50, key is test_row_0/A:col10/1732664007534/Put/seqid=0 2024-11-26T23:33:27,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741927_1103 (size=17181) 2024-11-26T23:33:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664067584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664067586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664067691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664067701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664067901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664067911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:27,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/535a25c6e617437ba590f33f58c2ecd1 2024-11-26T23:33:27,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/959552ffa61442e480134bed2c615d52 is 50, key is test_row_0/B:col10/1732664007534/Put/seqid=0 2024-11-26T23:33:27,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741928_1104 (size=12301) 2024-11-26T23:33:28,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664068212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664068223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:28,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/959552ffa61442e480134bed2c615d52 2024-11-26T23:33:28,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/99af886f8e0d46d39811ebf9b232b0ae is 50, key is test_row_0/C:col10/1732664007534/Put/seqid=0 2024-11-26T23:33:28,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741929_1105 (size=12301) 2024-11-26T23:33:28,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664068731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664068732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:28,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/99af886f8e0d46d39811ebf9b232b0ae 2024-11-26T23:33:28,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/535a25c6e617437ba590f33f58c2ecd1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/535a25c6e617437ba590f33f58c2ecd1 2024-11-26T23:33:28,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/535a25c6e617437ba590f33f58c2ecd1, entries=250, sequenceid=448, filesize=16.8 K 2024-11-26T23:33:28,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/959552ffa61442e480134bed2c615d52 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/959552ffa61442e480134bed2c615d52 2024-11-26T23:33:28,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/959552ffa61442e480134bed2c615d52, entries=150, sequenceid=448, filesize=12.0 K 2024-11-26T23:33:28,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/99af886f8e0d46d39811ebf9b232b0ae as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/99af886f8e0d46d39811ebf9b232b0ae 2024-11-26T23:33:28,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/99af886f8e0d46d39811ebf9b232b0ae, entries=150, sequenceid=448, filesize=12.0 K 2024-11-26T23:33:28,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 56adae88c925ec88c99616a57d431fab in 1270ms, sequenceid=448, compaction requested=true 2024-11-26T23:33:28,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:28,807 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:28,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:28,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:28,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:28,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:28,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:28,807 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:28,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:28,808 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:28,808 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:28,808 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:28,808 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/d6926de84212446abef54e3177462450, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/fac7b5740db541268b4034c8e76c57de, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/535a25c6e617437ba590f33f58c2ecd1] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=41.7 K 2024-11-26T23:33:28,808 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:28,809 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:28,809 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:28,809 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6926de84212446abef54e3177462450, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732664005179 2024-11-26T23:33:28,809 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9d3786be2ac042149ca05019eaba586a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a15ac53103624d64adfa4d9089bb947d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/959552ffa61442e480134bed2c615d52] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=37.0 K 2024-11-26T23:33:28,809 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac7b5740db541268b4034c8e76c57de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732664005223 2024-11-26T23:33:28,809 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3786be2ac042149ca05019eaba586a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732664005179 2024-11-26T23:33:28,810 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 535a25c6e617437ba590f33f58c2ecd1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732664007409 2024-11-26T23:33:28,810 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting a15ac53103624d64adfa4d9089bb947d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732664005223 2024-11-26T23:33:28,810 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 959552ffa61442e480134bed2c615d52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732664007422 2024-11-26T23:33:28,819 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:28,820 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/b1863e8a96134d9e887ff2de500e2ceb is 50, key is test_row_0/A:col10/1732664007534/Put/seqid=0 2024-11-26T23:33:28,822 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:28,823 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/53d45efc089642fa9f9a9cd7150913a5 is 50, key is test_row_0/B:col10/1732664007534/Put/seqid=0 2024-11-26T23:33:28,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741930_1106 (size=13357) 2024-11-26T23:33:28,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741931_1107 (size=13357) 2024-11-26T23:33:29,239 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/b1863e8a96134d9e887ff2de500e2ceb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b1863e8a96134d9e887ff2de500e2ceb 2024-11-26T23:33:29,246 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into b1863e8a96134d9e887ff2de500e2ceb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:29,246 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:29,247 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=13, startTime=1732664008807; duration=0sec 2024-11-26T23:33:29,247 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:29,247 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:29,247 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:29,248 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:29,248 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:29,248 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:29,248 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/eed6c5b570d64f739780cd532f19b02e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/44bfef45fee6496bba8c7b387377b51d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/99af886f8e0d46d39811ebf9b232b0ae] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=37.0 K 2024-11-26T23:33:29,249 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed6c5b570d64f739780cd532f19b02e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732664005179 2024-11-26T23:33:29,249 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44bfef45fee6496bba8c7b387377b51d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732664005223 2024-11-26T23:33:29,250 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99af886f8e0d46d39811ebf9b232b0ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732664007422 2024-11-26T23:33:29,258 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/53d45efc089642fa9f9a9cd7150913a5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/53d45efc089642fa9f9a9cd7150913a5 2024-11-26T23:33:29,284 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:29,286 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d779383b36214aa180b31683abe726ca is 50, key is test_row_0/C:col10/1732664007534/Put/seqid=0 2024-11-26T23:33:29,287 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into 53d45efc089642fa9f9a9cd7150913a5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:29,287 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:29,287 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=13, startTime=1732664008807; duration=0sec 2024-11-26T23:33:29,287 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:29,288 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:29,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741932_1108 (size=13357) 2024-11-26T23:33:29,306 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d779383b36214aa180b31683abe726ca as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d779383b36214aa180b31683abe726ca 2024-11-26T23:33:29,322 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into d779383b36214aa180b31683abe726ca(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:29,322 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:29,323 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=13, startTime=1732664008807; duration=0sec 2024-11-26T23:33:29,323 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:29,323 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T23:33:29,352 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:29,352 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-26T23:33:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T23:33:29,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:29,359 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:29,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T23:33:29,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-26T23:33:29,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:29,512 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-26T23:33:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/7dcc7a3d9411423e93781ed9e9ff6975 is 50, key is test_row_0/A:col10/1732664007583/Put/seqid=0 2024-11-26T23:33:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741933_1109 (size=12301) 2024-11-26T23:33:29,528 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/7dcc7a3d9411423e93781ed9e9ff6975 2024-11-26T23:33:29,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/afcd692444d145879978a2afc5a61674 is 50, key is test_row_0/B:col10/1732664007583/Put/seqid=0 2024-11-26T23:33:29,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741934_1110 (size=12301) 2024-11-26T23:33:29,559 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/afcd692444d145879978a2afc5a61674 2024-11-26T23:33:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/b7e9eac9913b4e559f4669901d2a7288 is 50, key is test_row_0/C:col10/1732664007583/Put/seqid=0 2024-11-26T23:33:29,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741935_1111 (size=12301) 2024-11-26T23:33:29,579 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/b7e9eac9913b4e559f4669901d2a7288 2024-11-26T23:33:29,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/7dcc7a3d9411423e93781ed9e9ff6975 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/7dcc7a3d9411423e93781ed9e9ff6975 2024-11-26T23:33:29,593 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/7dcc7a3d9411423e93781ed9e9ff6975, entries=150, sequenceid=473, filesize=12.0 K 2024-11-26T23:33:29,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/afcd692444d145879978a2afc5a61674 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/afcd692444d145879978a2afc5a61674 2024-11-26T23:33:29,601 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/afcd692444d145879978a2afc5a61674, entries=150, sequenceid=473, filesize=12.0 K 2024-11-26T23:33:29,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/b7e9eac9913b4e559f4669901d2a7288 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/b7e9eac9913b4e559f4669901d2a7288 2024-11-26T23:33:29,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/b7e9eac9913b4e559f4669901d2a7288, entries=150, sequenceid=473, filesize=12.0 K 2024-11-26T23:33:29,609 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 56adae88c925ec88c99616a57d431fab in 97ms, sequenceid=473, compaction requested=false 2024-11-26T23:33:29,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:29,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:29,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-26T23:33:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-26T23:33:29,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-26T23:33:29,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 254 msec 2024-11-26T23:33:29,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 264 msec 2024-11-26T23:33:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T23:33:29,672 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:29,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-26T23:33:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T23:33:29,674 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:29,675 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:29,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:29,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:29,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:29,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/2fb9d1f2f5c34f0c8c539cea3075e53e is 50, key is test_row_0/A:col10/1732664009756/Put/seqid=0 2024-11-26T23:33:29,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741936_1112 (size=17181) 2024-11-26T23:33:29,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/2fb9d1f2f5c34f0c8c539cea3075e53e 2024-11-26T23:33:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T23:33:29,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c1dc229edcbb4f17ad672cde1599ecb3 is 50, key is test_row_0/B:col10/1732664009756/Put/seqid=0 2024-11-26T23:33:29,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741937_1113 (size=12301) 2024-11-26T23:33:29,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664069824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:29,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664069825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:29,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T23:33:29,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:29,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:29,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:29,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:29,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664069931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:29,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664069932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:29,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T23:33:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:29,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T23:33:30,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T23:33:30,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:30,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664070141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:30,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664070141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:30,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c1dc229edcbb4f17ad672cde1599ecb3 2024-11-26T23:33:30,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d6ff41b04d6c48e8a2dc606f44d8dd43 is 50, key is test_row_0/C:col10/1732664009756/Put/seqid=0 2024-11-26T23:33:30,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741938_1114 (size=12301) 2024-11-26T23:33:30,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T23:33:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T23:33:30,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T23:33:30,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:30,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664070452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:30,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664070452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:30,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T23:33:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:30,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d6ff41b04d6c48e8a2dc606f44d8dd43 2024-11-26T23:33:30,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/2fb9d1f2f5c34f0c8c539cea3075e53e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/2fb9d1f2f5c34f0c8c539cea3075e53e 2024-11-26T23:33:30,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/2fb9d1f2f5c34f0c8c539cea3075e53e, entries=250, sequenceid=484, filesize=16.8 K 2024-11-26T23:33:30,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c1dc229edcbb4f17ad672cde1599ecb3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c1dc229edcbb4f17ad672cde1599ecb3 2024-11-26T23:33:30,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c1dc229edcbb4f17ad672cde1599ecb3, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T23:33:30,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d6ff41b04d6c48e8a2dc606f44d8dd43 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d6ff41b04d6c48e8a2dc606f44d8dd43 2024-11-26T23:33:30,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d6ff41b04d6c48e8a2dc606f44d8dd43, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T23:33:30,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 56adae88c925ec88c99616a57d431fab in 890ms, sequenceid=484, compaction requested=true 2024-11-26T23:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:30,647 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:30,647 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:30,648 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:30,648 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:30,648 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:30,648 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:30,649 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,649 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,649 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b1863e8a96134d9e887ff2de500e2ceb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/7dcc7a3d9411423e93781ed9e9ff6975, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/2fb9d1f2f5c34f0c8c539cea3075e53e] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=41.8 K 2024-11-26T23:33:30,649 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/53d45efc089642fa9f9a9cd7150913a5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/afcd692444d145879978a2afc5a61674, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c1dc229edcbb4f17ad672cde1599ecb3] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=37.1 K 2024-11-26T23:33:30,649 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1863e8a96134d9e887ff2de500e2ceb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732664007422 2024-11-26T23:33:30,649 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d45efc089642fa9f9a9cd7150913a5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732664007422 2024-11-26T23:33:30,650 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dcc7a3d9411423e93781ed9e9ff6975, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732664007572 2024-11-26T23:33:30,651 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting afcd692444d145879978a2afc5a61674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732664007572 2024-11-26T23:33:30,651 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb9d1f2f5c34f0c8c539cea3075e53e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732664009749 2024-11-26T23:33:30,652 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c1dc229edcbb4f17ad672cde1599ecb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732664009751 2024-11-26T23:33:30,659 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#103 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:30,659 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:30,660 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/9181711da9604b0ba20fbbea60b8fb1d is 50, key is test_row_0/B:col10/1732664009756/Put/seqid=0 2024-11-26T23:33:30,660 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/c19acc077cd649a487038e989baac355 is 50, key is test_row_0/A:col10/1732664009756/Put/seqid=0 2024-11-26T23:33:30,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741940_1116 (size=13459) 2024-11-26T23:33:30,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741939_1115 (size=13459) 2024-11-26T23:33:30,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T23:33:30,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:30,749 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:33:30,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/11c4ce56a56f4cc88a6e57cdc68767c2 is 50, key is test_row_0/A:col10/1732664009805/Put/seqid=0 2024-11-26T23:33:30,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741941_1117 (size=12301) 2024-11-26T23:33:30,762 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/11c4ce56a56f4cc88a6e57cdc68767c2 2024-11-26T23:33:30,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c46b748b2bd24718906e8f9d59c3ee68 is 50, key is test_row_0/B:col10/1732664009805/Put/seqid=0 2024-11-26T23:33:30,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741942_1118 (size=12301) 2024-11-26T23:33:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T23:33:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:30,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:30,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664070977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:30,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664070978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:31,073 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/c19acc077cd649a487038e989baac355 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c19acc077cd649a487038e989baac355 2024-11-26T23:33:31,079 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into c19acc077cd649a487038e989baac355(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:31,079 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:31,079 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=13, startTime=1732664010647; duration=0sec 2024-11-26T23:33:31,079 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:31,079 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:31,079 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:31,081 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:31,081 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:31,081 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:31,081 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d779383b36214aa180b31683abe726ca, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/b7e9eac9913b4e559f4669901d2a7288, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d6ff41b04d6c48e8a2dc606f44d8dd43] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=37.1 K 2024-11-26T23:33:31,082 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d779383b36214aa180b31683abe726ca, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732664007422 2024-11-26T23:33:31,082 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7e9eac9913b4e559f4669901d2a7288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732664007572 2024-11-26T23:33:31,083 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6ff41b04d6c48e8a2dc606f44d8dd43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732664009751 2024-11-26T23:33:31,089 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/9181711da9604b0ba20fbbea60b8fb1d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9181711da9604b0ba20fbbea60b8fb1d 2024-11-26T23:33:31,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:31,094 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664071092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:31,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664071092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:31,096 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d21769c459a844729e6518e49c3102df is 50, key is test_row_0/C:col10/1732664009756/Put/seqid=0 2024-11-26T23:33:31,097 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into 9181711da9604b0ba20fbbea60b8fb1d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:31,097 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:31,097 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=13, startTime=1732664010647; duration=0sec 2024-11-26T23:33:31,097 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:31,097 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:31,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741943_1119 (size=13459) 2024-11-26T23:33:31,177 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c46b748b2bd24718906e8f9d59c3ee68 2024-11-26T23:33:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ded778b8a7c24b35a0bdfb37161ae468 is 50, key is test_row_0/C:col10/1732664009805/Put/seqid=0 2024-11-26T23:33:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741944_1120 (size=12301) 2024-11-26T23:33:31,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664071302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:31,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664071302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:31,512 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/d21769c459a844729e6518e49c3102df as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d21769c459a844729e6518e49c3102df 2024-11-26T23:33:31,518 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into d21769c459a844729e6518e49c3102df(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:31,518 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:31,518 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=13, startTime=1732664010647; duration=0sec 2024-11-26T23:33:31,518 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:31,518 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:31,591 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ded778b8a7c24b35a0bdfb37161ae468 2024-11-26T23:33:31,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/11c4ce56a56f4cc88a6e57cdc68767c2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/11c4ce56a56f4cc88a6e57cdc68767c2 2024-11-26T23:33:31,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/11c4ce56a56f4cc88a6e57cdc68767c2, entries=150, sequenceid=509, filesize=12.0 K 2024-11-26T23:33:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/c46b748b2bd24718906e8f9d59c3ee68 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c46b748b2bd24718906e8f9d59c3ee68 2024-11-26T23:33:31,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664071611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:31,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664071611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:31,615 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c46b748b2bd24718906e8f9d59c3ee68, entries=150, sequenceid=509, filesize=12.0 K 2024-11-26T23:33:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ded778b8a7c24b35a0bdfb37161ae468 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ded778b8a7c24b35a0bdfb37161ae468 2024-11-26T23:33:31,624 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ded778b8a7c24b35a0bdfb37161ae468, entries=150, sequenceid=509, filesize=12.0 K 2024-11-26T23:33:31,628 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 56adae88c925ec88c99616a57d431fab in 878ms, sequenceid=509, compaction requested=false 2024-11-26T23:33:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:31,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-26T23:33:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-26T23:33:31,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-26T23:33:31,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-11-26T23:33:31,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.9630 sec 2024-11-26T23:33:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T23:33:31,822 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:31,822 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-26T23:33:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T23:33:31,825 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:31,829 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:31,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:31,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T23:33:31,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T23:33:31,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:31,982 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:31,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8b5e8379ca9444b299917998181d3527 is 50, key is test_row_0/A:col10/1732664010965/Put/seqid=0 2024-11-26T23:33:31,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741945_1121 (size=12301) 2024-11-26T23:33:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:32,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. as already flushing 2024-11-26T23:33:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T23:33:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664072168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664072170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664072281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664072281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,392 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8b5e8379ca9444b299917998181d3527 2024-11-26T23:33:32,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d8d6f69f59f64d9f95373407003c4bec is 50, key is test_row_0/B:col10/1732664010965/Put/seqid=0 2024-11-26T23:33:32,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741946_1122 (size=12301) 2024-11-26T23:33:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T23:33:32,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664072491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664072491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1732664072801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56426 deadline: 1732664072801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:33:32,806 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d8d6f69f59f64d9f95373407003c4bec 2024-11-26T23:33:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/faf8103a38444878babf5fa44481b611 is 50, key is test_row_0/C:col10/1732664010965/Put/seqid=0 2024-11-26T23:33:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741947_1123 (size=12301) 2024-11-26T23:33:32,821 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/faf8103a38444878babf5fa44481b611 2024-11-26T23:33:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/8b5e8379ca9444b299917998181d3527 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8b5e8379ca9444b299917998181d3527 2024-11-26T23:33:32,830 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8b5e8379ca9444b299917998181d3527, entries=150, sequenceid=523, filesize=12.0 K 2024-11-26T23:33:32,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/d8d6f69f59f64d9f95373407003c4bec as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d8d6f69f59f64d9f95373407003c4bec 2024-11-26T23:33:32,838 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d8d6f69f59f64d9f95373407003c4bec, entries=150, sequenceid=523, filesize=12.0 K 2024-11-26T23:33:32,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/faf8103a38444878babf5fa44481b611 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/faf8103a38444878babf5fa44481b611 2024-11-26T23:33:32,843 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/faf8103a38444878babf5fa44481b611, entries=150, sequenceid=523, filesize=12.0 K 2024-11-26T23:33:32,844 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 56adae88c925ec88c99616a57d431fab in 862ms, sequenceid=523, compaction requested=true 2024-11-26T23:33:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-26T23:33:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-26T23:33:32,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-26T23:33:32,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-26T23:33:32,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.0270 sec 2024-11-26T23:33:32,954 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-26T23:33:32,954 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:32,956 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-26T23:33:32,956 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-26T23:33:32,956 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:32,956 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:32,957 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-26T23:33:32,957 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:32,959 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,959 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,959 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,959 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,959 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,959 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:32,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:32,960 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:32,960 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:32,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T23:33:32,962 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:33,314 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:33,314 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-26T23:33:33,314 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:33,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:33:33,314 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-26T23:33:33,314 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:33,314 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:33,314 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:33,314 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:33,314 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:33,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:33,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:33,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/912c03d262c54dfeb204bc12527ad55e is 50, key is test_row_0/A:col10/1732664013312/Put/seqid=0 2024-11-26T23:33:33,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741948_1124 (size=12301) 2024-11-26T23:33:33,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/912c03d262c54dfeb204bc12527ad55e 2024-11-26T23:33:33,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f479f67ddacb4a4eb9642b9ae37cd535 is 50, key is test_row_0/B:col10/1732664013312/Put/seqid=0 2024-11-26T23:33:33,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741949_1125 (size=12301) 2024-11-26T23:33:34,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f479f67ddacb4a4eb9642b9ae37cd535 2024-11-26T23:33:34,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/48dac3f437184a969ca5019dd864423f is 50, key is test_row_0/C:col10/1732664013312/Put/seqid=0 2024-11-26T23:33:34,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741950_1126 (size=12301) 2024-11-26T23:33:34,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/48dac3f437184a969ca5019dd864423f 2024-11-26T23:33:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/912c03d262c54dfeb204bc12527ad55e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/912c03d262c54dfeb204bc12527ad55e 2024-11-26T23:33:34,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/912c03d262c54dfeb204bc12527ad55e, entries=150, sequenceid=550, filesize=12.0 K 2024-11-26T23:33:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/f479f67ddacb4a4eb9642b9ae37cd535 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f479f67ddacb4a4eb9642b9ae37cd535 2024-11-26T23:33:34,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f479f67ddacb4a4eb9642b9ae37cd535, entries=150, sequenceid=550, filesize=12.0 K 2024-11-26T23:33:34,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/48dac3f437184a969ca5019dd864423f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/48dac3f437184a969ca5019dd864423f 2024-11-26T23:33:34,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/48dac3f437184a969ca5019dd864423f, entries=150, sequenceid=550, filesize=12.0 K 2024-11-26T23:33:34,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=0 B/0 for 56adae88c925ec88c99616a57d431fab in 1251ms, sequenceid=550, compaction requested=true 2024-11-26T23:33:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:34,565 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:34,565 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56adae88c925ec88c99616a57d431fab:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:34,566 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:34,566 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:34,567 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/A is initiating minor compaction (all files) 2024-11-26T23:33:34,567 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/B is initiating minor compaction (all files) 2024-11-26T23:33:34,567 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/A in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:34,567 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/B in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:34,567 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c19acc077cd649a487038e989baac355, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/11c4ce56a56f4cc88a6e57cdc68767c2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8b5e8379ca9444b299917998181d3527, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/912c03d262c54dfeb204bc12527ad55e] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=49.2 K 2024-11-26T23:33:34,567 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9181711da9604b0ba20fbbea60b8fb1d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c46b748b2bd24718906e8f9d59c3ee68, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d8d6f69f59f64d9f95373407003c4bec, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f479f67ddacb4a4eb9642b9ae37cd535] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=49.2 K 2024-11-26T23:33:34,567 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19acc077cd649a487038e989baac355, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732664009751 2024-11-26T23:33:34,567 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9181711da9604b0ba20fbbea60b8fb1d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732664009751 2024-11-26T23:33:34,567 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c4ce56a56f4cc88a6e57cdc68767c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1732664009805 2024-11-26T23:33:34,567 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c46b748b2bd24718906e8f9d59c3ee68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1732664009805 2024-11-26T23:33:34,568 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b5e8379ca9444b299917998181d3527, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732664010965 2024-11-26T23:33:34,568 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d6f69f59f64d9f95373407003c4bec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732664010965 2024-11-26T23:33:34,568 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f479f67ddacb4a4eb9642b9ae37cd535, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1732664012162 2024-11-26T23:33:34,568 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 912c03d262c54dfeb204bc12527ad55e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1732664012162 2024-11-26T23:33:34,577 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#A#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:34,578 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/193154e1f3114bfd871e2d2d407b40ba is 50, key is test_row_0/A:col10/1732664013312/Put/seqid=0 2024-11-26T23:33:34,578 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#B#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:34,579 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a26ca09064a54031a1bfb0527fe0f569 is 50, key is test_row_0/B:col10/1732664013312/Put/seqid=0 2024-11-26T23:33:34,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741951_1127 (size=13595) 2024-11-26T23:33:34,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741952_1128 (size=13595) 2024-11-26T23:33:34,596 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:33:34,987 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/193154e1f3114bfd871e2d2d407b40ba as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/193154e1f3114bfd871e2d2d407b40ba 2024-11-26T23:33:34,989 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/a26ca09064a54031a1bfb0527fe0f569 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a26ca09064a54031a1bfb0527fe0f569 2024-11-26T23:33:34,993 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/A of 56adae88c925ec88c99616a57d431fab into 193154e1f3114bfd871e2d2d407b40ba(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:34,993 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:34,993 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/A, priority=12, startTime=1732664014565; duration=0sec 2024-11-26T23:33:34,993 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:34,993 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:A 2024-11-26T23:33:34,993 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:34,994 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:34,994 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/B of 56adae88c925ec88c99616a57d431fab into a26ca09064a54031a1bfb0527fe0f569(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:34,995 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 56adae88c925ec88c99616a57d431fab/C is initiating minor compaction (all files) 2024-11-26T23:33:34,995 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:34,995 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/B, priority=12, startTime=1732664014565; duration=0sec 2024-11-26T23:33:34,995 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56adae88c925ec88c99616a57d431fab/C in TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:34,995 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:34,995 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:B 2024-11-26T23:33:34,995 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d21769c459a844729e6518e49c3102df, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ded778b8a7c24b35a0bdfb37161ae468, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/faf8103a38444878babf5fa44481b611, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/48dac3f437184a969ca5019dd864423f] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp, totalSize=49.2 K 2024-11-26T23:33:34,995 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d21769c459a844729e6518e49c3102df, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732664009751 2024-11-26T23:33:34,995 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded778b8a7c24b35a0bdfb37161ae468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1732664009805 2024-11-26T23:33:34,996 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting faf8103a38444878babf5fa44481b611, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732664010965 2024-11-26T23:33:34,996 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48dac3f437184a969ca5019dd864423f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1732664012162 2024-11-26T23:33:35,004 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56adae88c925ec88c99616a57d431fab#C#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:35,004 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ccec70087539479da31a5e37c8d42d37 is 50, key is test_row_0/C:col10/1732664013312/Put/seqid=0 2024-11-26T23:33:35,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741953_1129 (size=13595) 2024-11-26T23:33:35,414 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/ccec70087539479da31a5e37c8d42d37 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ccec70087539479da31a5e37c8d42d37 2024-11-26T23:33:35,418 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56adae88c925ec88c99616a57d431fab/C of 56adae88c925ec88c99616a57d431fab into ccec70087539479da31a5e37c8d42d37(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:35,419 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56adae88c925ec88c99616a57d431fab: 2024-11-26T23:33:35,419 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., storeName=56adae88c925ec88c99616a57d431fab/C, priority=12, startTime=1732664014565; duration=0sec 2024-11-26T23:33:35,419 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:35,419 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56adae88c925ec88c99616a57d431fab:C 2024-11-26T23:33:35,973 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-26T23:33:35,973 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:35,973 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:35,973 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:35,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:36,013 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-26T23:33:36,013 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:36,013 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:36,013 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:36,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:36,022 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-26T23:33:36,023 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:33:36,023 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:36,023 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:36,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 182 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3589 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3451 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1556 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4658 rows 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1567 2024-11-26T23:33:36,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4700 rows 2024-11-26T23:33:36,023 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:33:36,024 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:33:36,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:36,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:36,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:33:36,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T23:33:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T23:33:36,035 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664016034"}]},"ts":"1732664016034"} 2024-11-26T23:33:36,036 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T23:33:36,036 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T23:33:36,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:33:36,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, UNASSIGN}] 2024-11-26T23:33:36,042 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, UNASSIGN 2024-11-26T23:33:36,043 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=56adae88c925ec88c99616a57d431fab, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:36,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, UNASSIGN because future has completed 2024-11-26T23:33:36,045 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:33:36,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:33:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T23:33:36,201 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:36,202 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:33:36,202 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 56adae88c925ec88c99616a57d431fab, disabling compactions & flushes 2024-11-26T23:33:36,202 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:36,202 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:36,202 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. after waiting 0 ms 2024-11-26T23:33:36,202 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:36,203 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 56adae88c925ec88c99616a57d431fab 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-26T23:33:36,207 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/c182c47e88e7488c8472254ef8881783 is 50, key is test_row_1/A:col10/1732664016021/Put/seqid=0 2024-11-26T23:33:36,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741954_1130 (size=9857) 2024-11-26T23:33:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T23:33:36,611 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/c182c47e88e7488c8472254ef8881783 2024-11-26T23:33:36,618 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/2a442bf34cd54d6a8f7271d794abcdd5 is 50, key is test_row_1/B:col10/1732664016021/Put/seqid=0 2024-11-26T23:33:36,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741955_1131 (size=9857) 2024-11-26T23:33:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T23:33:37,023 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/2a442bf34cd54d6a8f7271d794abcdd5 2024-11-26T23:33:37,030 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/aaddb276207140639123ee14396f7a09 is 50, key is test_row_1/C:col10/1732664016021/Put/seqid=0 2024-11-26T23:33:37,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741956_1132 (size=9857) 2024-11-26T23:33:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T23:33:37,435 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/aaddb276207140639123ee14396f7a09 2024-11-26T23:33:37,440 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/A/c182c47e88e7488c8472254ef8881783 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c182c47e88e7488c8472254ef8881783 2024-11-26T23:33:37,443 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c182c47e88e7488c8472254ef8881783, entries=100, sequenceid=559, filesize=9.6 K 2024-11-26T23:33:37,444 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/B/2a442bf34cd54d6a8f7271d794abcdd5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2a442bf34cd54d6a8f7271d794abcdd5 2024-11-26T23:33:37,448 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2a442bf34cd54d6a8f7271d794abcdd5, entries=100, sequenceid=559, filesize=9.6 K 2024-11-26T23:33:37,448 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/.tmp/C/aaddb276207140639123ee14396f7a09 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aaddb276207140639123ee14396f7a09 2024-11-26T23:33:37,452 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aaddb276207140639123ee14396f7a09, entries=100, sequenceid=559, filesize=9.6 K 2024-11-26T23:33:37,453 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 56adae88c925ec88c99616a57d431fab in 1251ms, sequenceid=559, compaction requested=false 2024-11-26T23:33:37,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/203a3fe210eb49879534e68a3508ec1f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8615b0a309e145ec909009896cdedad4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/134f125058e44da79375e666c1664414, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b938a5707f484fa7b338ef4daa003035, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/03e5d365ffff42f6831acd7082e5276e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/539a5157586544acae7d6f61392077bf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/0d9c430f711e4b708e3a2f2b39187020, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f80eacab35894f94a1b09dfc4bfb1a6e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/475687b876bb46b2bfa27ac7df4343ec, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6fcc21d1379649e484b952203bcaea12, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f1161d9256134cb98009d0d053440097, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/9ac152ebf36244619b0b5fcf4d64ef80, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/4ad9064807a842db87c2c5b2065c3a55, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/36bc41183ee7407db0b1a000983be6fe, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e1f7a92a1d4e4148b8f348ae61a9202e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/3bc8ff2748504e9a9f346d82f68dd239, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/5a255c5660da41ddb47eb9bfd932f15c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f83a3efff8a2457fa7a9cc763ab90591, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e7bda0cbd0474ac68b7a303683f16e8e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e4df861bb3cd44f38ff61f842e265274, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8570353ff780474792fefde0afc3f3bb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/efeb3ffe89344e259aebe14f37a681f3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ccbbc686e38849b2b2c031d7edd9202c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/bb790fd98d8a403fa6acfbd2803648d2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/470919b9399640e2ba8990e1851fab85, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ecb72eb43cad4747a64d89c99fe03b71, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6642e82c756f40e5be1392262be8bd03, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/d6926de84212446abef54e3177462450, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6021c605195a4a55a356cd3feb98b053, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/fac7b5740db541268b4034c8e76c57de, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/535a25c6e617437ba590f33f58c2ecd1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b1863e8a96134d9e887ff2de500e2ceb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/7dcc7a3d9411423e93781ed9e9ff6975, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/2fb9d1f2f5c34f0c8c539cea3075e53e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c19acc077cd649a487038e989baac355, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/11c4ce56a56f4cc88a6e57cdc68767c2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8b5e8379ca9444b299917998181d3527, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/912c03d262c54dfeb204bc12527ad55e] to archive 2024-11-26T23:33:37,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:33:37,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/203a3fe210eb49879534e68a3508ec1f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/203a3fe210eb49879534e68a3508ec1f 2024-11-26T23:33:37,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8615b0a309e145ec909009896cdedad4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8615b0a309e145ec909009896cdedad4 2024-11-26T23:33:37,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/134f125058e44da79375e666c1664414 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/134f125058e44da79375e666c1664414 2024-11-26T23:33:37,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b938a5707f484fa7b338ef4daa003035 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b938a5707f484fa7b338ef4daa003035 2024-11-26T23:33:37,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/03e5d365ffff42f6831acd7082e5276e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/03e5d365ffff42f6831acd7082e5276e 2024-11-26T23:33:37,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/539a5157586544acae7d6f61392077bf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/539a5157586544acae7d6f61392077bf 2024-11-26T23:33:37,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/0d9c430f711e4b708e3a2f2b39187020 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/0d9c430f711e4b708e3a2f2b39187020 2024-11-26T23:33:37,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f80eacab35894f94a1b09dfc4bfb1a6e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f80eacab35894f94a1b09dfc4bfb1a6e 2024-11-26T23:33:37,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/475687b876bb46b2bfa27ac7df4343ec to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/475687b876bb46b2bfa27ac7df4343ec 2024-11-26T23:33:37,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6fcc21d1379649e484b952203bcaea12 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6fcc21d1379649e484b952203bcaea12 2024-11-26T23:33:37,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f1161d9256134cb98009d0d053440097 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f1161d9256134cb98009d0d053440097 2024-11-26T23:33:37,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/9ac152ebf36244619b0b5fcf4d64ef80 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/9ac152ebf36244619b0b5fcf4d64ef80 2024-11-26T23:33:37,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/4ad9064807a842db87c2c5b2065c3a55 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/4ad9064807a842db87c2c5b2065c3a55 2024-11-26T23:33:37,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/36bc41183ee7407db0b1a000983be6fe to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/36bc41183ee7407db0b1a000983be6fe 2024-11-26T23:33:37,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e1f7a92a1d4e4148b8f348ae61a9202e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e1f7a92a1d4e4148b8f348ae61a9202e 2024-11-26T23:33:37,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/3bc8ff2748504e9a9f346d82f68dd239 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/3bc8ff2748504e9a9f346d82f68dd239 2024-11-26T23:33:37,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/5a255c5660da41ddb47eb9bfd932f15c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/5a255c5660da41ddb47eb9bfd932f15c 2024-11-26T23:33:37,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f83a3efff8a2457fa7a9cc763ab90591 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/f83a3efff8a2457fa7a9cc763ab90591 2024-11-26T23:33:37,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e7bda0cbd0474ac68b7a303683f16e8e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e7bda0cbd0474ac68b7a303683f16e8e 2024-11-26T23:33:37,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e4df861bb3cd44f38ff61f842e265274 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/e4df861bb3cd44f38ff61f842e265274 2024-11-26T23:33:37,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8570353ff780474792fefde0afc3f3bb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8570353ff780474792fefde0afc3f3bb 2024-11-26T23:33:37,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/efeb3ffe89344e259aebe14f37a681f3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/efeb3ffe89344e259aebe14f37a681f3 2024-11-26T23:33:37,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ccbbc686e38849b2b2c031d7edd9202c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ccbbc686e38849b2b2c031d7edd9202c 2024-11-26T23:33:37,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/bb790fd98d8a403fa6acfbd2803648d2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/bb790fd98d8a403fa6acfbd2803648d2 2024-11-26T23:33:37,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/470919b9399640e2ba8990e1851fab85 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/470919b9399640e2ba8990e1851fab85 2024-11-26T23:33:37,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ecb72eb43cad4747a64d89c99fe03b71 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/ecb72eb43cad4747a64d89c99fe03b71 2024-11-26T23:33:37,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6642e82c756f40e5be1392262be8bd03 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6642e82c756f40e5be1392262be8bd03 2024-11-26T23:33:37,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/d6926de84212446abef54e3177462450 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/d6926de84212446abef54e3177462450 2024-11-26T23:33:37,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6021c605195a4a55a356cd3feb98b053 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/6021c605195a4a55a356cd3feb98b053 2024-11-26T23:33:37,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/fac7b5740db541268b4034c8e76c57de to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/fac7b5740db541268b4034c8e76c57de 2024-11-26T23:33:37,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/535a25c6e617437ba590f33f58c2ecd1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/535a25c6e617437ba590f33f58c2ecd1 2024-11-26T23:33:37,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b1863e8a96134d9e887ff2de500e2ceb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/b1863e8a96134d9e887ff2de500e2ceb 2024-11-26T23:33:37,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/7dcc7a3d9411423e93781ed9e9ff6975 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/7dcc7a3d9411423e93781ed9e9ff6975 2024-11-26T23:33:37,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/2fb9d1f2f5c34f0c8c539cea3075e53e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/2fb9d1f2f5c34f0c8c539cea3075e53e 2024-11-26T23:33:37,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c19acc077cd649a487038e989baac355 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c19acc077cd649a487038e989baac355 2024-11-26T23:33:37,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/11c4ce56a56f4cc88a6e57cdc68767c2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/11c4ce56a56f4cc88a6e57cdc68767c2 2024-11-26T23:33:37,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8b5e8379ca9444b299917998181d3527 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/8b5e8379ca9444b299917998181d3527 2024-11-26T23:33:37,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/912c03d262c54dfeb204bc12527ad55e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/912c03d262c54dfeb204bc12527ad55e 2024-11-26T23:33:37,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/65c0f4c303284d3f885e775b7f6f1b6d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dc6c26f64a2a4606b94ccbb11beedf48, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d41c80ab4d254bb6809f5d09ed92522b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/de7c5382df70452d8bea775ab7288eb1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/0f320b90f2d24672bc36927fb46d22aa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d60042642a7e4c509d4a46a3602a0d92, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/06b845f8fd3a4cd9a4b2462ac1bba2fb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/248a2574204740f2a3bd02a70a411d6f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/4798b740479249f9bb3f6f8a2f5dc8fa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e846dedf84a14e64b57f04059e22a654, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/b03781781cda4870a2b7f774f681bb9e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f04918b4a6ac4e9494605f68fac20a28, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/466e7408e6e44bf29e5bfcdee3412884, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c20584933a3443c89d871c34c1a54985, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/fb598b11f46441139fe7e14f6d618bc6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e26f05b602ac4f90b62e584358846cc4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dab6ce1062874ed8a9d7bd7a10ef975e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/88613be832cd4bd68829c1407fa75f7b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/404fe875109f45738b2e48be63ffb168, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c6cdfaace84a452583216bed7a51b9fb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9e7dfbdb2d8c4927a2fd33b8070d232b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/76780be895364a939fc26b3e269784aa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2edc56f621b64097805dcee4bf739d7e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d75c53cae7874f139224aaa9b06d4a08, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e51020dc501446e092a89a1298cab761, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f3f2bd0d11994c519ac7fd0fc2dc6917, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a353a907041040cb9824bba301c2f36c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9d3786be2ac042149ca05019eaba586a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/503c1dee3b5b4372a26db6ce1b772f93, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a15ac53103624d64adfa4d9089bb947d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/53d45efc089642fa9f9a9cd7150913a5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/959552ffa61442e480134bed2c615d52, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/afcd692444d145879978a2afc5a61674, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9181711da9604b0ba20fbbea60b8fb1d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c1dc229edcbb4f17ad672cde1599ecb3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c46b748b2bd24718906e8f9d59c3ee68, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d8d6f69f59f64d9f95373407003c4bec, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f479f67ddacb4a4eb9642b9ae37cd535] to archive 2024-11-26T23:33:37,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:33:37,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/65c0f4c303284d3f885e775b7f6f1b6d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/65c0f4c303284d3f885e775b7f6f1b6d 2024-11-26T23:33:37,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dc6c26f64a2a4606b94ccbb11beedf48 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dc6c26f64a2a4606b94ccbb11beedf48 2024-11-26T23:33:37,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d41c80ab4d254bb6809f5d09ed92522b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d41c80ab4d254bb6809f5d09ed92522b 2024-11-26T23:33:37,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/de7c5382df70452d8bea775ab7288eb1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/de7c5382df70452d8bea775ab7288eb1 2024-11-26T23:33:37,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/0f320b90f2d24672bc36927fb46d22aa to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/0f320b90f2d24672bc36927fb46d22aa 2024-11-26T23:33:37,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d60042642a7e4c509d4a46a3602a0d92 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d60042642a7e4c509d4a46a3602a0d92 2024-11-26T23:33:37,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/06b845f8fd3a4cd9a4b2462ac1bba2fb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/06b845f8fd3a4cd9a4b2462ac1bba2fb 2024-11-26T23:33:37,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/248a2574204740f2a3bd02a70a411d6f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/248a2574204740f2a3bd02a70a411d6f 2024-11-26T23:33:37,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/4798b740479249f9bb3f6f8a2f5dc8fa to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/4798b740479249f9bb3f6f8a2f5dc8fa 2024-11-26T23:33:37,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e846dedf84a14e64b57f04059e22a654 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e846dedf84a14e64b57f04059e22a654 2024-11-26T23:33:37,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/b03781781cda4870a2b7f774f681bb9e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/b03781781cda4870a2b7f774f681bb9e 2024-11-26T23:33:37,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f04918b4a6ac4e9494605f68fac20a28 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f04918b4a6ac4e9494605f68fac20a28 2024-11-26T23:33:37,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/466e7408e6e44bf29e5bfcdee3412884 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/466e7408e6e44bf29e5bfcdee3412884 2024-11-26T23:33:37,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c20584933a3443c89d871c34c1a54985 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c20584933a3443c89d871c34c1a54985 2024-11-26T23:33:37,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/fb598b11f46441139fe7e14f6d618bc6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/fb598b11f46441139fe7e14f6d618bc6 2024-11-26T23:33:37,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e26f05b602ac4f90b62e584358846cc4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e26f05b602ac4f90b62e584358846cc4 2024-11-26T23:33:37,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dab6ce1062874ed8a9d7bd7a10ef975e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/dab6ce1062874ed8a9d7bd7a10ef975e 2024-11-26T23:33:37,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/88613be832cd4bd68829c1407fa75f7b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/88613be832cd4bd68829c1407fa75f7b 2024-11-26T23:33:37,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/404fe875109f45738b2e48be63ffb168 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/404fe875109f45738b2e48be63ffb168 2024-11-26T23:33:37,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c6cdfaace84a452583216bed7a51b9fb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c6cdfaace84a452583216bed7a51b9fb 2024-11-26T23:33:37,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9e7dfbdb2d8c4927a2fd33b8070d232b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9e7dfbdb2d8c4927a2fd33b8070d232b 2024-11-26T23:33:37,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/76780be895364a939fc26b3e269784aa to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/76780be895364a939fc26b3e269784aa 2024-11-26T23:33:37,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2edc56f621b64097805dcee4bf739d7e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2edc56f621b64097805dcee4bf739d7e 2024-11-26T23:33:37,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d75c53cae7874f139224aaa9b06d4a08 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d75c53cae7874f139224aaa9b06d4a08 2024-11-26T23:33:37,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e51020dc501446e092a89a1298cab761 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/e51020dc501446e092a89a1298cab761 2024-11-26T23:33:37,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f3f2bd0d11994c519ac7fd0fc2dc6917 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f3f2bd0d11994c519ac7fd0fc2dc6917 2024-11-26T23:33:37,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a353a907041040cb9824bba301c2f36c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a353a907041040cb9824bba301c2f36c 2024-11-26T23:33:37,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9d3786be2ac042149ca05019eaba586a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9d3786be2ac042149ca05019eaba586a 2024-11-26T23:33:37,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/503c1dee3b5b4372a26db6ce1b772f93 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/503c1dee3b5b4372a26db6ce1b772f93 2024-11-26T23:33:37,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a15ac53103624d64adfa4d9089bb947d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a15ac53103624d64adfa4d9089bb947d 2024-11-26T23:33:37,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/53d45efc089642fa9f9a9cd7150913a5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/53d45efc089642fa9f9a9cd7150913a5 2024-11-26T23:33:37,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/959552ffa61442e480134bed2c615d52 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/959552ffa61442e480134bed2c615d52 2024-11-26T23:33:37,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/afcd692444d145879978a2afc5a61674 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/afcd692444d145879978a2afc5a61674 2024-11-26T23:33:37,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9181711da9604b0ba20fbbea60b8fb1d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/9181711da9604b0ba20fbbea60b8fb1d 2024-11-26T23:33:37,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c1dc229edcbb4f17ad672cde1599ecb3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c1dc229edcbb4f17ad672cde1599ecb3 2024-11-26T23:33:37,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c46b748b2bd24718906e8f9d59c3ee68 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/c46b748b2bd24718906e8f9d59c3ee68 2024-11-26T23:33:37,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d8d6f69f59f64d9f95373407003c4bec to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/d8d6f69f59f64d9f95373407003c4bec 2024-11-26T23:33:37,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f479f67ddacb4a4eb9642b9ae37cd535 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/f479f67ddacb4a4eb9642b9ae37cd535 2024-11-26T23:33:37,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a0d975266a2d4101b79187263483ac7e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f78a9da310af42c189f5763b1bc5db2a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/36a99eb252004fb08f7462be4edbe02d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d0770b46a436454abf95196ab3a70931, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/38539fdd26544b988076c583a59e3b77, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/61e15f046c664e858726dd5fedba9c0d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6e4f8516866142be9603dfeeb8b7f30e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a1f2f537463a481b9eebbab85447b393, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/5554575179434d0ca02f756b493fb62a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/dc1fb09efb174a26bc03eb5fadef2682, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6b3dbaab13c74b02b410a064fe3629ff, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aa303a5e741d41abbcd7689b7c3c6d6d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/81d611c5ccc84cd6a6a63544b06e9d95, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/1c1394374d9b48269ef35f6ed2d95fa6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ff3d4ebeb2f74681a4c4e78be79a13b3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/8b3be3a2fe924374bd7cd29d5b43ece3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f17f16c584b743a9aa9638f4e0538867, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d15d8193e1ef42e8bc2844c8ab21c7b7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/88878b403da4420b808a0e32f7762404, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/3afa672130d74188bc894239842a5a0f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/76ef7923405e4ff0a8e74f650cbd44c2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/46157877a4e640f7b76ea3cbf57e4a96, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ca3ceb89ce06491690986b2570389977, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/80f3286126de4bbf975606c6ff1433a9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d3dd391d5b1245e38232ca9c2bb41e14, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/782bcd6a2d5f494ba52ea984150150a0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/60805fea63c347d0b1e1a022ed464607, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/eed6c5b570d64f739780cd532f19b02e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/90259e47496649bbb8564ef21aa5350d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/44bfef45fee6496bba8c7b387377b51d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d779383b36214aa180b31683abe726ca, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/99af886f8e0d46d39811ebf9b232b0ae, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/b7e9eac9913b4e559f4669901d2a7288, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d21769c459a844729e6518e49c3102df, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d6ff41b04d6c48e8a2dc606f44d8dd43, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ded778b8a7c24b35a0bdfb37161ae468, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/faf8103a38444878babf5fa44481b611, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/48dac3f437184a969ca5019dd864423f] to archive 2024-11-26T23:33:37,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:33:37,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a0d975266a2d4101b79187263483ac7e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a0d975266a2d4101b79187263483ac7e 2024-11-26T23:33:37,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f78a9da310af42c189f5763b1bc5db2a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f78a9da310af42c189f5763b1bc5db2a 2024-11-26T23:33:37,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/36a99eb252004fb08f7462be4edbe02d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/36a99eb252004fb08f7462be4edbe02d 2024-11-26T23:33:37,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d0770b46a436454abf95196ab3a70931 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d0770b46a436454abf95196ab3a70931 2024-11-26T23:33:37,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/38539fdd26544b988076c583a59e3b77 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/38539fdd26544b988076c583a59e3b77 2024-11-26T23:33:37,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/61e15f046c664e858726dd5fedba9c0d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/61e15f046c664e858726dd5fedba9c0d 2024-11-26T23:33:37,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6e4f8516866142be9603dfeeb8b7f30e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6e4f8516866142be9603dfeeb8b7f30e 2024-11-26T23:33:37,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a1f2f537463a481b9eebbab85447b393 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/a1f2f537463a481b9eebbab85447b393 2024-11-26T23:33:37,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/5554575179434d0ca02f756b493fb62a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/5554575179434d0ca02f756b493fb62a 2024-11-26T23:33:37,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/dc1fb09efb174a26bc03eb5fadef2682 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/dc1fb09efb174a26bc03eb5fadef2682 2024-11-26T23:33:37,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6b3dbaab13c74b02b410a064fe3629ff to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/6b3dbaab13c74b02b410a064fe3629ff 2024-11-26T23:33:37,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aa303a5e741d41abbcd7689b7c3c6d6d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aa303a5e741d41abbcd7689b7c3c6d6d 2024-11-26T23:33:37,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/81d611c5ccc84cd6a6a63544b06e9d95 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/81d611c5ccc84cd6a6a63544b06e9d95 2024-11-26T23:33:37,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/1c1394374d9b48269ef35f6ed2d95fa6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/1c1394374d9b48269ef35f6ed2d95fa6 2024-11-26T23:33:37,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ff3d4ebeb2f74681a4c4e78be79a13b3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ff3d4ebeb2f74681a4c4e78be79a13b3 2024-11-26T23:33:37,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/8b3be3a2fe924374bd7cd29d5b43ece3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/8b3be3a2fe924374bd7cd29d5b43ece3 2024-11-26T23:33:37,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f17f16c584b743a9aa9638f4e0538867 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/f17f16c584b743a9aa9638f4e0538867 2024-11-26T23:33:37,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d15d8193e1ef42e8bc2844c8ab21c7b7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d15d8193e1ef42e8bc2844c8ab21c7b7 2024-11-26T23:33:37,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/88878b403da4420b808a0e32f7762404 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/88878b403da4420b808a0e32f7762404 2024-11-26T23:33:37,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/3afa672130d74188bc894239842a5a0f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/3afa672130d74188bc894239842a5a0f 2024-11-26T23:33:37,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/76ef7923405e4ff0a8e74f650cbd44c2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/76ef7923405e4ff0a8e74f650cbd44c2 2024-11-26T23:33:37,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/46157877a4e640f7b76ea3cbf57e4a96 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/46157877a4e640f7b76ea3cbf57e4a96 2024-11-26T23:33:37,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ca3ceb89ce06491690986b2570389977 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ca3ceb89ce06491690986b2570389977 2024-11-26T23:33:37,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/80f3286126de4bbf975606c6ff1433a9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/80f3286126de4bbf975606c6ff1433a9 2024-11-26T23:33:37,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d3dd391d5b1245e38232ca9c2bb41e14 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d3dd391d5b1245e38232ca9c2bb41e14 2024-11-26T23:33:37,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/782bcd6a2d5f494ba52ea984150150a0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/782bcd6a2d5f494ba52ea984150150a0 2024-11-26T23:33:37,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/60805fea63c347d0b1e1a022ed464607 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/60805fea63c347d0b1e1a022ed464607 2024-11-26T23:33:37,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/eed6c5b570d64f739780cd532f19b02e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/eed6c5b570d64f739780cd532f19b02e 2024-11-26T23:33:37,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/90259e47496649bbb8564ef21aa5350d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/90259e47496649bbb8564ef21aa5350d 2024-11-26T23:33:37,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/44bfef45fee6496bba8c7b387377b51d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/44bfef45fee6496bba8c7b387377b51d 2024-11-26T23:33:37,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d779383b36214aa180b31683abe726ca to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d779383b36214aa180b31683abe726ca 2024-11-26T23:33:37,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/99af886f8e0d46d39811ebf9b232b0ae to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/99af886f8e0d46d39811ebf9b232b0ae 2024-11-26T23:33:37,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/b7e9eac9913b4e559f4669901d2a7288 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/b7e9eac9913b4e559f4669901d2a7288 2024-11-26T23:33:37,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d21769c459a844729e6518e49c3102df to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d21769c459a844729e6518e49c3102df 2024-11-26T23:33:37,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d6ff41b04d6c48e8a2dc606f44d8dd43 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/d6ff41b04d6c48e8a2dc606f44d8dd43 2024-11-26T23:33:37,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ded778b8a7c24b35a0bdfb37161ae468 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ded778b8a7c24b35a0bdfb37161ae468 2024-11-26T23:33:37,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/faf8103a38444878babf5fa44481b611 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/faf8103a38444878babf5fa44481b611 2024-11-26T23:33:37,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/48dac3f437184a969ca5019dd864423f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/48dac3f437184a969ca5019dd864423f 2024-11-26T23:33:37,615 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/recovered.edits/562.seqid, newMaxSeqId=562, maxSeqId=1 2024-11-26T23:33:37,615 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. 2024-11-26T23:33:37,615 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 56adae88c925ec88c99616a57d431fab: Waiting for close lock at 1732664016202Running coprocessor pre-close hooks at 1732664016202Disabling compacts and flushes for region at 1732664016202Disabling writes for close at 1732664016202Obtaining lock to block concurrent updates at 1732664016203 (+1 ms)Preparing flush snapshotting stores in 56adae88c925ec88c99616a57d431fab at 1732664016203Finished memstore snapshotting TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732664016203Flushing stores of TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab. at 1732664016203Flushing 56adae88c925ec88c99616a57d431fab/A: creating writer at 1732664016203Flushing 56adae88c925ec88c99616a57d431fab/A: appending metadata at 1732664016206 (+3 ms)Flushing 56adae88c925ec88c99616a57d431fab/A: closing flushed file at 1732664016206Flushing 56adae88c925ec88c99616a57d431fab/B: creating writer at 1732664016615 (+409 ms)Flushing 56adae88c925ec88c99616a57d431fab/B: appending metadata at 1732664016618 (+3 ms)Flushing 56adae88c925ec88c99616a57d431fab/B: closing flushed file at 1732664016618Flushing 56adae88c925ec88c99616a57d431fab/C: creating writer at 1732664017028 (+410 ms)Flushing 56adae88c925ec88c99616a57d431fab/C: appending metadata at 1732664017030 (+2 ms)Flushing 56adae88c925ec88c99616a57d431fab/C: closing flushed file at 1732664017030Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6edd5844: reopening flushed file at 1732664017439 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ab26902: reopening flushed file at 1732664017444 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e9d1da0: reopening flushed file at 1732664017448 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 56adae88c925ec88c99616a57d431fab in 1251ms, sequenceid=559, compaction requested=false at 1732664017453 (+5 ms)Writing region close event to WAL at 1732664017611 (+158 ms)Running coprocessor post-close hooks at 1732664017615 (+4 ms)Closed at 1732664017615 2024-11-26T23:33:37,618 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:37,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=56adae88c925ec88c99616a57d431fab, regionState=CLOSED 2024-11-26T23:33:37,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:33:37,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-26T23:33:37,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 56adae88c925ec88c99616a57d431fab, server=c707e9ace675,43077,1732663986534 in 1.5760 sec 2024-11-26T23:33:37,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-26T23:33:37,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56adae88c925ec88c99616a57d431fab, UNASSIGN in 1.5810 sec 2024-11-26T23:33:37,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-26T23:33:37,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5870 sec 2024-11-26T23:33:37,628 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664017627"}]},"ts":"1732664017627"} 2024-11-26T23:33:37,629 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T23:33:37,629 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T23:33:37,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6010 sec 2024-11-26T23:33:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T23:33:38,192 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:38,195 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T23:33:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:38,200 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T23:33:38,202 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:38,206 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:38,210 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/recovered.edits] 2024-11-26T23:33:38,213 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/193154e1f3114bfd871e2d2d407b40ba to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/193154e1f3114bfd871e2d2d407b40ba 2024-11-26T23:33:38,214 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c182c47e88e7488c8472254ef8881783 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/A/c182c47e88e7488c8472254ef8881783 2024-11-26T23:33:38,216 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2a442bf34cd54d6a8f7271d794abcdd5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/2a442bf34cd54d6a8f7271d794abcdd5 2024-11-26T23:33:38,217 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a26ca09064a54031a1bfb0527fe0f569 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/B/a26ca09064a54031a1bfb0527fe0f569 2024-11-26T23:33:38,219 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aaddb276207140639123ee14396f7a09 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/aaddb276207140639123ee14396f7a09 2024-11-26T23:33:38,221 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ccec70087539479da31a5e37c8d42d37 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/C/ccec70087539479da31a5e37c8d42d37 2024-11-26T23:33:38,224 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/recovered.edits/562.seqid to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab/recovered.edits/562.seqid 2024-11-26T23:33:38,224 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/56adae88c925ec88c99616a57d431fab 2024-11-26T23:33:38,224 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T23:33:38,229 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:38,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-26T23:33:38,238 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T23:33:38,273 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T23:33:38,274 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:38,274 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T23:33:38,275 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732664018275"}]},"ts":"9223372036854775807"} 2024-11-26T23:33:38,279 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T23:33:38,279 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 56adae88c925ec88c99616a57d431fab, NAME => 'TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T23:33:38,279 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T23:33:38,279 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732664018279"}]},"ts":"9223372036854775807"} 2024-11-26T23:33:38,282 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T23:33:38,283 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:38,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-11-26T23:33:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T23:33:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T23:33:38,313 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:38,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_560703445_22 at /127.0.0.1:56620 [Waiting for operation #284] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_560703445_22 at /127.0.0.1:49490 [Waiting for operation #194] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c707e9ace675:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c707e9ace675:43077-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1770722268_22 at /127.0.0.1:49262 [Waiting for operation #266] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 265) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6823 (was 7450) 2024-11-26T23:33:38,331 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=6823 2024-11-26T23:33:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:33:38,333 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:33:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:38,335 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:33:38,335 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:38,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-26T23:33:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T23:33:38,336 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:33:38,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741957_1133 (size=959) 2024-11-26T23:33:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T23:33:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T23:33:38,744 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:33:38,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741958_1134 (size=53) 2024-11-26T23:33:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T23:33:39,150 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:39,150 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2b323bd2428037d76aaf73c5c85f5d46, disabling compactions & flushes 2024-11-26T23:33:39,150 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:39,150 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:39,150 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. after waiting 0 ms 2024-11-26T23:33:39,150 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:39,150 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:39,150 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2b323bd2428037d76aaf73c5c85f5d46: Waiting for close lock at 1732664019150Disabling compacts and flushes for region at 1732664019150Disabling writes for close at 1732664019150Writing region close event to WAL at 1732664019150Closed at 1732664019150 2024-11-26T23:33:39,151 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:33:39,152 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732664019151"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732664019151"}]},"ts":"1732664019151"} 2024-11-26T23:33:39,153 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:33:39,154 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:33:39,154 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664019154"}]},"ts":"1732664019154"} 2024-11-26T23:33:39,155 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T23:33:39,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, ASSIGN}] 2024-11-26T23:33:39,157 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, ASSIGN 2024-11-26T23:33:39,157 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, ASSIGN; state=OFFLINE, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=false 2024-11-26T23:33:39,308 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:39,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, ASSIGN because future has completed 2024-11-26T23:33:39,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:33:39,466 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:39,466 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:33:39,466 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,466 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:39,466 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,466 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,468 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,470 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b323bd2428037d76aaf73c5c85f5d46 columnFamilyName A 2024-11-26T23:33:39,470 DEBUG [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:39,471 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(327): Store=2b323bd2428037d76aaf73c5c85f5d46/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:39,471 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T23:33:39,472 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b323bd2428037d76aaf73c5c85f5d46 columnFamilyName B 2024-11-26T23:33:39,472 DEBUG [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:39,473 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(327): Store=2b323bd2428037d76aaf73c5c85f5d46/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:39,473 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,473 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b323bd2428037d76aaf73c5c85f5d46 columnFamilyName C 2024-11-26T23:33:39,474 DEBUG [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:39,474 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(327): Store=2b323bd2428037d76aaf73c5c85f5d46/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:39,474 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,475 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,475 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,476 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,476 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,476 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:33:39,477 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,479 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:33:39,479 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 2b323bd2428037d76aaf73c5c85f5d46; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74588615, jitterRate=0.1114569753408432}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:33:39,479 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:39,480 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 2b323bd2428037d76aaf73c5c85f5d46: Running coprocessor pre-open hook at 1732664019467Writing region info on filesystem at 1732664019467Initializing all the Stores at 1732664019467Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664019467Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664019468 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664019468Cleaning up temporary data from old regions at 1732664019476 (+8 ms)Running coprocessor post-open hooks at 1732664019479 (+3 ms)Region opened successfully at 1732664019480 (+1 ms) 2024-11-26T23:33:39,481 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., pid=36, masterSystemTime=1732664019462 2024-11-26T23:33:39,483 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:39,483 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:39,483 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:39,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:33:39,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-26T23:33:39,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 in 175 msec 2024-11-26T23:33:39,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-26T23:33:39,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, ASSIGN in 331 msec 2024-11-26T23:33:39,490 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:33:39,490 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664019490"}]},"ts":"1732664019490"} 2024-11-26T23:33:39,491 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T23:33:39,492 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:33:39,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-26T23:33:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T23:33:40,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:40,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a2f40f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:40,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:40,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:40,484 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d4c072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:40,486 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b76b47a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:40,487 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:40,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:40,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:33:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35833,1732663985624 2024-11-26T23:33:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73b9f97d 2024-11-26T23:33:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:33:40,493 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:33:40,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:33:40,497 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:33:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T23:33:40,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741959_1135 (size=995) 2024-11-26T23:33:40,915 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-26T23:33:40,915 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-26T23:33:40,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:33:40,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, REOPEN/MOVE}] 2024-11-26T23:33:40,926 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, REOPEN/MOVE 2024-11-26T23:33:40,927 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:40,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, REOPEN/MOVE because future has completed 2024-11-26T23:33:40,929 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:33:40,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:33:41,081 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,081 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:33:41,081 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 2b323bd2428037d76aaf73c5c85f5d46, disabling compactions & flushes 2024-11-26T23:33:41,081 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,081 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,081 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. after waiting 0 ms 2024-11-26T23:33:41,081 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,085 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T23:33:41,086 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,086 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 2b323bd2428037d76aaf73c5c85f5d46: Waiting for close lock at 1732664021081Running coprocessor pre-close hooks at 1732664021081Disabling compacts and flushes for region at 1732664021081Disabling writes for close at 1732664021081Writing region close event to WAL at 1732664021082 (+1 ms)Running coprocessor post-close hooks at 1732664021086 (+4 ms)Closed at 1732664021086 2024-11-26T23:33:41,086 WARN [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 2b323bd2428037d76aaf73c5c85f5d46 to self. 2024-11-26T23:33:41,088 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,088 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=CLOSED 2024-11-26T23:33:41,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:33:41,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-26T23:33:41,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 in 162 msec 2024-11-26T23:33:41,093 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, REOPEN/MOVE; state=CLOSED, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=true 2024-11-26T23:33:41,243 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, REOPEN/MOVE because future has completed 2024-11-26T23:33:41,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:33:41,400 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,401 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:33:41,401 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,401 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:33:41,401 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,401 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,405 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,411 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b323bd2428037d76aaf73c5c85f5d46 columnFamilyName A 2024-11-26T23:33:41,412 DEBUG [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:41,413 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(327): Store=2b323bd2428037d76aaf73c5c85f5d46/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:41,413 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,414 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b323bd2428037d76aaf73c5c85f5d46 columnFamilyName B 2024-11-26T23:33:41,414 DEBUG [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:41,415 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(327): Store=2b323bd2428037d76aaf73c5c85f5d46/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:41,415 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,415 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b323bd2428037d76aaf73c5c85f5d46 columnFamilyName C 2024-11-26T23:33:41,416 DEBUG [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:41,416 INFO [StoreOpener-2b323bd2428037d76aaf73c5c85f5d46-1 {}] regionserver.HStore(327): Store=2b323bd2428037d76aaf73c5c85f5d46/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:33:41,416 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,417 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,418 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,418 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,418 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,419 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:33:41,420 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,421 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 2b323bd2428037d76aaf73c5c85f5d46; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64497045, jitterRate=-0.03891913592815399}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:33:41,421 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,422 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 2b323bd2428037d76aaf73c5c85f5d46: Running coprocessor pre-open hook at 1732664021401Writing region info on filesystem at 1732664021401Initializing all the Stores at 1732664021402 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664021402Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664021402Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664021402Cleaning up temporary data from old regions at 1732664021418 (+16 ms)Running coprocessor post-open hooks at 1732664021421 (+3 ms)Region opened successfully at 1732664021422 (+1 ms) 2024-11-26T23:33:41,423 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., pid=41, masterSystemTime=1732664021397 2024-11-26T23:33:41,424 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,424 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,425 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=OPEN, openSeqNum=5, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:33:41,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-26T23:33:41,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 in 182 msec 2024-11-26T23:33:41,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-26T23:33:41,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, REOPEN/MOVE in 504 msec 2024-11-26T23:33:41,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-26T23:33:41,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-26T23:33:41,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 935 msec 2024-11-26T23:33:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-26T23:33:41,438 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:41,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,439 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba48b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,441 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20562f95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,443 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677da74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,444 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6c4c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,446 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40581156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,448 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8be20f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,450 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3954a2e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5b1946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,453 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559b6e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,454 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f52ffb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,456 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62acf3b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,457 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a5e6bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,459 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a746049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,460 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3202d9a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,462 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c0cc05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,463 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0dd047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711c9f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:33:41,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:33:41,465 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:33:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:33:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:33:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d9a762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:33:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:33:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:33:41,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:33:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3fb0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:33:41,467 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,467 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,468 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:41,468 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,469 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,469 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,470 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,470 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,480 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:33:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-26T23:33:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,481 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:33:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:33:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T23:33:41,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:33:41,483 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:41,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:33:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:41,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:41,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ec1c471fd1854737b4d87b09a7f66984_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664021490/Put/seqid=0 2024-11-26T23:33:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664081547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664081548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664081548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664081552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664081553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741960_1136 (size=9714) 2024-11-26T23:33:41,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T23:33:41,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ec1c471fd1854737b4d87b09a7f66984_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec1c471fd1854737b4d87b09a7f66984_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:41,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/915e9de783f341f99eb6e8eb68c95308, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:41,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/915e9de783f341f99eb6e8eb68c95308 is 175, key is test_row_0/A:col10/1732664021490/Put/seqid=0 2024-11-26T23:33:41,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741961_1137 (size=22361) 2024-11-26T23:33:41,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/915e9de783f341f99eb6e8eb68c95308 2024-11-26T23:33:41,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:41,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:41,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/8c7ee126b092434faa8fea9d4f8cf402 is 50, key is test_row_0/B:col10/1732664021490/Put/seqid=0 2024-11-26T23:33:41,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664081665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664081665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664081665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664081666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664081666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741962_1138 (size=9657) 2024-11-26T23:33:41,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/8c7ee126b092434faa8fea9d4f8cf402 2024-11-26T23:33:41,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/62bef4aa710c4702a9f411a149f3b829 is 50, key is test_row_0/C:col10/1732664021490/Put/seqid=0 2024-11-26T23:33:41,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741963_1139 (size=9657) 2024-11-26T23:33:41,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T23:33:41,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664081881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664081882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664081882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664081882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664081882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:41,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:41,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T23:33:42,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/62bef4aa710c4702a9f411a149f3b829 2024-11-26T23:33:42,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/915e9de783f341f99eb6e8eb68c95308 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/915e9de783f341f99eb6e8eb68c95308 2024-11-26T23:33:42,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/915e9de783f341f99eb6e8eb68c95308, entries=100, sequenceid=15, filesize=21.8 K 2024-11-26T23:33:42,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/8c7ee126b092434faa8fea9d4f8cf402 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/8c7ee126b092434faa8fea9d4f8cf402 2024-11-26T23:33:42,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/8c7ee126b092434faa8fea9d4f8cf402, entries=100, sequenceid=15, filesize=9.4 K 2024-11-26T23:33:42,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/62bef4aa710c4702a9f411a149f3b829 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/62bef4aa710c4702a9f411a149f3b829 2024-11-26T23:33:42,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/62bef4aa710c4702a9f411a149f3b829, entries=100, sequenceid=15, filesize=9.4 K 2024-11-26T23:33:42,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2b323bd2428037d76aaf73c5c85f5d46 in 680ms, sequenceid=15, compaction requested=false 2024-11-26T23:33:42,172 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T23:33:42,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:42,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T23:33:42,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664082205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664082205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262ce790652f5242369eadcb9941c99375_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664022197/Put/seqid=0 2024-11-26T23:33:42,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664082207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664082211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664082213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741964_1140 (size=17034) 2024-11-26T23:33:42,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262ce790652f5242369eadcb9941c99375_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262ce790652f5242369eadcb9941c99375_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:42,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/0f41af5ba87041ba9eddbe552d5faf48, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:42,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/0f41af5ba87041ba9eddbe552d5faf48 is 175, key is test_row_0/A:col10/1732664022197/Put/seqid=0 2024-11-26T23:33:42,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741965_1141 (size=48139) 2024-11-26T23:33:42,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/0f41af5ba87041ba9eddbe552d5faf48 2024-11-26T23:33:42,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/74edd45ea1084816bea619d2a3195345 is 50, key is test_row_0/B:col10/1732664022197/Put/seqid=0 2024-11-26T23:33:42,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741966_1142 (size=12001) 2024-11-26T23:33:42,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/74edd45ea1084816bea619d2a3195345 2024-11-26T23:33:42,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664082322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664082322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664082323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/fe28f94dc0ba4e899debecd0a4487a6b is 50, key is test_row_0/C:col10/1732664022197/Put/seqid=0 2024-11-26T23:33:42,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664082332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664082332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741967_1143 (size=12001) 2024-11-26T23:33:42,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664082532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664082532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664082532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664082542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664082542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:42,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T23:33:42,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:42,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/fe28f94dc0ba4e899debecd0a4487a6b 2024-11-26T23:33:42,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/0f41af5ba87041ba9eddbe552d5faf48 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/0f41af5ba87041ba9eddbe552d5faf48 2024-11-26T23:33:42,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/0f41af5ba87041ba9eddbe552d5faf48, entries=250, sequenceid=42, filesize=47.0 K 2024-11-26T23:33:42,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/74edd45ea1084816bea619d2a3195345 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/74edd45ea1084816bea619d2a3195345 2024-11-26T23:33:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/74edd45ea1084816bea619d2a3195345, entries=150, sequenceid=42, filesize=11.7 K 2024-11-26T23:33:42,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/fe28f94dc0ba4e899debecd0a4487a6b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/fe28f94dc0ba4e899debecd0a4487a6b 2024-11-26T23:33:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/fe28f94dc0ba4e899debecd0a4487a6b, entries=150, sequenceid=42, filesize=11.7 K 2024-11-26T23:33:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 2b323bd2428037d76aaf73c5c85f5d46 in 593ms, sequenceid=42, compaction requested=false 2024-11-26T23:33:42,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T23:33:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:42,872 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T23:33:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c8a7a1e6e8d24b29b2425a6a7579b33c_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_1/A:col10/1732664022867/Put/seqid=0 2024-11-26T23:33:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741968_1144 (size=12154) 2024-11-26T23:33:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664083085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664083090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664083095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664083099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664083100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,153 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:33:43,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664083212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664083212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664083222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664083222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664083223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:43,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c8a7a1e6e8d24b29b2425a6a7579b33c_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8a7a1e6e8d24b29b2425a6a7579b33c_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:43,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/3ff6754e165e4192a26743a00e8ffca8, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:43,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/3ff6754e165e4192a26743a00e8ffca8 is 175, key is test_row_1/A:col10/1732664022867/Put/seqid=0 2024-11-26T23:33:43,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741969_1145 (size=30951) 2024-11-26T23:33:43,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664083422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664083422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664083432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664083432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664083432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T23:33:43,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664083732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664083742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664083742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664083742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664083742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:43,764 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/3ff6754e165e4192a26743a00e8ffca8 2024-11-26T23:33:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/71ad390683e94543b24ac3fa36d1c2cf is 50, key is test_row_1/B:col10/1732664022867/Put/seqid=0 2024-11-26T23:33:43,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741970_1146 (size=9657) 2024-11-26T23:33:44,190 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/71ad390683e94543b24ac3fa36d1c2cf 2024-11-26T23:33:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/2bbff5fda9ae4a44beb240a7f594e40e is 50, key is test_row_1/C:col10/1732664022867/Put/seqid=0 2024-11-26T23:33:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741971_1147 (size=9657) 2024-11-26T23:33:44,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664084242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:44,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664084251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664084251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664084251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664084252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:44,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/2bbff5fda9ae4a44beb240a7f594e40e 2024-11-26T23:33:44,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/3ff6754e165e4192a26743a00e8ffca8 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/3ff6754e165e4192a26743a00e8ffca8 2024-11-26T23:33:44,615 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/3ff6754e165e4192a26743a00e8ffca8, entries=150, sequenceid=52, filesize=30.2 K 2024-11-26T23:33:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/71ad390683e94543b24ac3fa36d1c2cf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/71ad390683e94543b24ac3fa36d1c2cf 2024-11-26T23:33:44,620 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/71ad390683e94543b24ac3fa36d1c2cf, entries=100, sequenceid=52, filesize=9.4 K 2024-11-26T23:33:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/2bbff5fda9ae4a44beb240a7f594e40e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2bbff5fda9ae4a44beb240a7f594e40e 2024-11-26T23:33:44,626 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2bbff5fda9ae4a44beb240a7f594e40e, entries=100, sequenceid=52, filesize=9.4 K 2024-11-26T23:33:44,628 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 2b323bd2428037d76aaf73c5c85f5d46 in 1755ms, sequenceid=52, compaction requested=true 2024-11-26T23:33:44,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:44,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:44,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-26T23:33:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-26T23:33:44,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T23:33:44,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1460 sec 2024-11-26T23:33:44,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.1640 sec 2024-11-26T23:33:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:45,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-26T23:33:45,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268895761d238d4f1c9ebcaec18b407d63_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664023094/Put/seqid=0 2024-11-26T23:33:45,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664085277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664085279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664085279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664085279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664085279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741972_1148 (size=14594) 2024-11-26T23:33:45,298 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:45,303 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268895761d238d4f1c9ebcaec18b407d63_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268895761d238d4f1c9ebcaec18b407d63_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:45,306 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/e9c40f34bcd14e32bd25ca6c0795a84f, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:45,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/e9c40f34bcd14e32bd25ca6c0795a84f is 175, key is test_row_0/A:col10/1732664023094/Put/seqid=0 2024-11-26T23:33:45,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741973_1149 (size=39549) 2024-11-26T23:33:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664085392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664085392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664085392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664085392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664085392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664085602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664085602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664085602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664085602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664085602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T23:33:45,642 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:45,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-26T23:33:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T23:33:45,647 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:45,648 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:45,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:45,716 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/e9c40f34bcd14e32bd25ca6c0795a84f 2024-11-26T23:33:45,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0 is 50, key is test_row_0/B:col10/1732664023094/Put/seqid=0 2024-11-26T23:33:45,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741974_1150 (size=12001) 2024-11-26T23:33:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T23:33:45,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T23:33:45,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:45,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:45,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:45,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:45,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:45,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664085911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664085911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664085911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664085911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664085911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:45,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T23:33:45,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:45,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:45,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:45,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:45,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T23:33:46,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T23:33:46,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:46,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0 2024-11-26T23:33:46,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/c702e81503af4f7cbc806636b3a0d574 is 50, key is test_row_0/C:col10/1732664023094/Put/seqid=0 2024-11-26T23:33:46,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741975_1151 (size=12001) 2024-11-26T23:33:46,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T23:33:46,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:46,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T23:33:46,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T23:33:46,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:46,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664086422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664086422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664086422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664086422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664086422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:46,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/c702e81503af4f7cbc806636b3a0d574 2024-11-26T23:33:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/e9c40f34bcd14e32bd25ca6c0795a84f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/e9c40f34bcd14e32bd25ca6c0795a84f 2024-11-26T23:33:46,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/e9c40f34bcd14e32bd25ca6c0795a84f, entries=200, sequenceid=80, filesize=38.6 K 2024-11-26T23:33:46,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T23:33:46,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:46,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0 2024-11-26T23:33:46,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:46,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T23:33:46,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/c702e81503af4f7cbc806636b3a0d574 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/c702e81503af4f7cbc806636b3a0d574 2024-11-26T23:33:46,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/c702e81503af4f7cbc806636b3a0d574, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T23:33:46,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 2b323bd2428037d76aaf73c5c85f5d46 in 1317ms, sequenceid=80, compaction requested=true 2024-11-26T23:33:46,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:46,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:46,582 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:46,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:46,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:46,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:46,582 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:46,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:46,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:46,584 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:46,584 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/B is initiating minor compaction (all files) 2024-11-26T23:33:46,584 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/B in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,584 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/8c7ee126b092434faa8fea9d4f8cf402, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/74edd45ea1084816bea619d2a3195345, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/71ad390683e94543b24ac3fa36d1c2cf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=42.3 K 2024-11-26T23:33:46,584 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141000 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:46,585 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/A is initiating minor compaction (all files) 2024-11-26T23:33:46,585 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/A in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,585 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/915e9de783f341f99eb6e8eb68c95308, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/0f41af5ba87041ba9eddbe552d5faf48, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/3ff6754e165e4192a26743a00e8ffca8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/e9c40f34bcd14e32bd25ca6c0795a84f] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=137.7 K 2024-11-26T23:33:46,585 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7ee126b092434faa8fea9d4f8cf402, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732664021484 2024-11-26T23:33:46,585 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,585 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/915e9de783f341f99eb6e8eb68c95308, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/0f41af5ba87041ba9eddbe552d5faf48, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/3ff6754e165e4192a26743a00e8ffca8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/e9c40f34bcd14e32bd25ca6c0795a84f] 2024-11-26T23:33:46,586 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 74edd45ea1084816bea619d2a3195345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732664021548 2024-11-26T23:33:46,586 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ad390683e94543b24ac3fa36d1c2cf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664022210 2024-11-26T23:33:46,587 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb93cfdd4ae4c85ba13a0d5b3bec3d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664023089 2024-11-26T23:33:46,588 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915e9de783f341f99eb6e8eb68c95308, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732664021484 2024-11-26T23:33:46,589 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f41af5ba87041ba9eddbe552d5faf48, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732664021544 2024-11-26T23:33:46,590 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ff6754e165e4192a26743a00e8ffca8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664022206 2024-11-26T23:33:46,590 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9c40f34bcd14e32bd25ca6c0795a84f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664023089 2024-11-26T23:33:46,600 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:46,601 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/6974491dba1d49ceb32fbb971a08ca35 is 50, key is test_row_0/B:col10/1732664023094/Put/seqid=0 2024-11-26T23:33:46,630 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:46,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741976_1152 (size=12139) 2024-11-26T23:33:46,635 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126f7a2b42b52124e8eb70a1471754b0e1e_2b323bd2428037d76aaf73c5c85f5d46 store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:46,638 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126f7a2b42b52124e8eb70a1471754b0e1e_2b323bd2428037d76aaf73c5c85f5d46, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:46,638 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/6974491dba1d49ceb32fbb971a08ca35 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/6974491dba1d49ceb32fbb971a08ca35 2024-11-26T23:33:46,639 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f7a2b42b52124e8eb70a1471754b0e1e_2b323bd2428037d76aaf73c5c85f5d46 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:46,647 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/B of 2b323bd2428037d76aaf73c5c85f5d46 into 6974491dba1d49ceb32fbb971a08ca35(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:46,647 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:46,647 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/B, priority=12, startTime=1732664026582; duration=0sec 2024-11-26T23:33:46,647 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:46,647 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:B 2024-11-26T23:33:46,647 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:46,649 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:46,650 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/C is initiating minor compaction (all files) 2024-11-26T23:33:46,650 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/C in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,650 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/62bef4aa710c4702a9f411a149f3b829, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/fe28f94dc0ba4e899debecd0a4487a6b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2bbff5fda9ae4a44beb240a7f594e40e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/c702e81503af4f7cbc806636b3a0d574] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=42.3 K 2024-11-26T23:33:46,650 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 62bef4aa710c4702a9f411a149f3b829, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732664021484 2024-11-26T23:33:46,651 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting fe28f94dc0ba4e899debecd0a4487a6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732664021548 2024-11-26T23:33:46,652 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbff5fda9ae4a44beb240a7f594e40e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664022210 2024-11-26T23:33:46,653 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c702e81503af4f7cbc806636b3a0d574, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664023089 2024-11-26T23:33:46,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741977_1153 (size=4469) 2024-11-26T23:33:46,656 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#A#compaction#133 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:46,657 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/55195552f75e46a9bbbb228deeb741c5 is 175, key is test_row_0/A:col10/1732664023094/Put/seqid=0 2024-11-26T23:33:46,666 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:46,667 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/0c9732d0c68b4b58a932e0a54b366a51 is 50, key is test_row_0/C:col10/1732664023094/Put/seqid=0 2024-11-26T23:33:46,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741978_1154 (size=31093) 2024-11-26T23:33:46,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741979_1155 (size=12139) 2024-11-26T23:33:46,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T23:33:46,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:46,721 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-26T23:33:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e2f19a735f8c4502a767481cf30b515c_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664025276/Put/seqid=0 2024-11-26T23:33:46,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741980_1156 (size=12154) 2024-11-26T23:33:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T23:33:47,077 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/55195552f75e46a9bbbb228deeb741c5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/55195552f75e46a9bbbb228deeb741c5 2024-11-26T23:33:47,083 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/A of 2b323bd2428037d76aaf73c5c85f5d46 into 55195552f75e46a9bbbb228deeb741c5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:47,083 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:47,083 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/A, priority=12, startTime=1732664026582; duration=0sec 2024-11-26T23:33:47,083 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:47,083 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:A 2024-11-26T23:33:47,107 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/0c9732d0c68b4b58a932e0a54b366a51 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/0c9732d0c68b4b58a932e0a54b366a51 2024-11-26T23:33:47,113 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/C of 2b323bd2428037d76aaf73c5c85f5d46 into 0c9732d0c68b4b58a932e0a54b366a51(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:47,114 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:47,114 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/C, priority=12, startTime=1732664026582; duration=0sec 2024-11-26T23:33:47,114 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:47,114 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:C 2024-11-26T23:33:47,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:47,142 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e2f19a735f8c4502a767481cf30b515c_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e2f19a735f8c4502a767481cf30b515c_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:47,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/d2d1db480929488eab9f463eac67b97e, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:47,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/d2d1db480929488eab9f463eac67b97e is 175, key is test_row_0/A:col10/1732664025276/Put/seqid=0 2024-11-26T23:33:47,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741981_1157 (size=30955) 2024-11-26T23:33:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:47,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:47,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664087468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664087468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664087469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664087470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664087476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,549 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/d2d1db480929488eab9f463eac67b97e 2024-11-26T23:33:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b8641d866723435aa682a9419a487ea5 is 50, key is test_row_0/B:col10/1732664025276/Put/seqid=0 2024-11-26T23:33:47,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741982_1158 (size=12001) 2024-11-26T23:33:47,569 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b8641d866723435aa682a9419a487ea5 2024-11-26T23:33:47,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/36d73214b0ba404f8e6f8d41d5d91eed is 50, key is test_row_0/C:col10/1732664025276/Put/seqid=0 2024-11-26T23:33:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664087581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664087581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664087582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664087582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741983_1159 (size=12001) 2024-11-26T23:33:47,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664087592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T23:33:47,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664087791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664087791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664087792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664087792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664087801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:47,991 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/36d73214b0ba404f8e6f8d41d5d91eed 2024-11-26T23:33:47,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/d2d1db480929488eab9f463eac67b97e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/d2d1db480929488eab9f463eac67b97e 2024-11-26T23:33:48,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/d2d1db480929488eab9f463eac67b97e, entries=150, sequenceid=90, filesize=30.2 K 2024-11-26T23:33:48,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b8641d866723435aa682a9419a487ea5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b8641d866723435aa682a9419a487ea5 2024-11-26T23:33:48,006 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b8641d866723435aa682a9419a487ea5, entries=150, sequenceid=90, filesize=11.7 K 2024-11-26T23:33:48,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/36d73214b0ba404f8e6f8d41d5d91eed as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/36d73214b0ba404f8e6f8d41d5d91eed 2024-11-26T23:33:48,011 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/36d73214b0ba404f8e6f8d41d5d91eed, entries=150, sequenceid=90, filesize=11.7 K 2024-11-26T23:33:48,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 2b323bd2428037d76aaf73c5c85f5d46 in 1292ms, sequenceid=90, compaction requested=false 2024-11-26T23:33:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-26T23:33:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-26T23:33:48,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-26T23:33:48,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3670 sec 2024-11-26T23:33:48,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3750 sec 2024-11-26T23:33:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:48,105 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:48,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-26T23:33:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664088108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664088110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664088111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263fe889d15dec489b8f972e363443fa41_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664028103/Put/seqid=0 2024-11-26T23:33:48,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664088114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664088114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741984_1160 (size=14594) 2024-11-26T23:33:48,122 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:48,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263fe889d15dec489b8f972e363443fa41_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263fe889d15dec489b8f972e363443fa41_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:48,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/caa8474b01564ebe8e60573e1a8de657, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:48,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/caa8474b01564ebe8e60573e1a8de657 is 175, key is test_row_0/A:col10/1732664028103/Put/seqid=0 2024-11-26T23:33:48,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741985_1161 (size=39549) 2024-11-26T23:33:48,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664088221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664088221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664088221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664088222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664088431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664088431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664088431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664088431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,534 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/caa8474b01564ebe8e60573e1a8de657 2024-11-26T23:33:48,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/fff4cdfe68ee45c1a5713596bbf2d332 is 50, key is test_row_0/B:col10/1732664028103/Put/seqid=0 2024-11-26T23:33:48,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741986_1162 (size=12001) 2024-11-26T23:33:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664088621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664088741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664088741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664088741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664088741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:48,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/fff4cdfe68ee45c1a5713596bbf2d332 2024-11-26T23:33:48,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/a69a96a4bea54d0a83fd5163801628c2 is 50, key is test_row_0/C:col10/1732664028103/Put/seqid=0 2024-11-26T23:33:48,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741987_1163 (size=12001) 2024-11-26T23:33:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664089251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664089251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664089251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:49,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664089262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:49,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/a69a96a4bea54d0a83fd5163801628c2 2024-11-26T23:33:49,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/caa8474b01564ebe8e60573e1a8de657 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/caa8474b01564ebe8e60573e1a8de657 2024-11-26T23:33:49,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/caa8474b01564ebe8e60573e1a8de657, entries=200, sequenceid=120, filesize=38.6 K 2024-11-26T23:33:49,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/fff4cdfe68ee45c1a5713596bbf2d332 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/fff4cdfe68ee45c1a5713596bbf2d332 2024-11-26T23:33:49,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/fff4cdfe68ee45c1a5713596bbf2d332, entries=150, sequenceid=120, filesize=11.7 K 2024-11-26T23:33:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/a69a96a4bea54d0a83fd5163801628c2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/a69a96a4bea54d0a83fd5163801628c2 2024-11-26T23:33:49,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/a69a96a4bea54d0a83fd5163801628c2, entries=150, sequenceid=120, filesize=11.7 K 2024-11-26T23:33:49,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 2b323bd2428037d76aaf73c5c85f5d46 in 1283ms, sequenceid=120, compaction requested=true 2024-11-26T23:33:49,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:49,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:49,389 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:49,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:49,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:49,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:49,389 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:49,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:49,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:49,390 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:49,390 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:49,390 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/A is initiating minor compaction (all files) 2024-11-26T23:33:49,391 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/B is initiating minor compaction (all files) 2024-11-26T23:33:49,391 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/B in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:49,391 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/A in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:49,391 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/6974491dba1d49ceb32fbb971a08ca35, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b8641d866723435aa682a9419a487ea5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/fff4cdfe68ee45c1a5713596bbf2d332] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=35.3 K 2024-11-26T23:33:49,391 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/55195552f75e46a9bbbb228deeb741c5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/d2d1db480929488eab9f463eac67b97e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/caa8474b01564ebe8e60573e1a8de657] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=99.2 K 2024-11-26T23:33:49,391 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:49,391 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/55195552f75e46a9bbbb228deeb741c5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/d2d1db480929488eab9f463eac67b97e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/caa8474b01564ebe8e60573e1a8de657] 2024-11-26T23:33:49,391 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55195552f75e46a9bbbb228deeb741c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664023089 2024-11-26T23:33:49,391 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6974491dba1d49ceb32fbb971a08ca35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664023089 2024-11-26T23:33:49,392 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d1db480929488eab9f463eac67b97e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732664025276 2024-11-26T23:33:49,392 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b8641d866723435aa682a9419a487ea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732664025276 2024-11-26T23:33:49,392 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa8474b01564ebe8e60573e1a8de657, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732664027466 2024-11-26T23:33:49,393 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting fff4cdfe68ee45c1a5713596bbf2d332, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732664027466 2024-11-26T23:33:49,417 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:49,418 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/130bdcd09139476b9600f810bf27ca5a is 50, key is test_row_0/B:col10/1732664028103/Put/seqid=0 2024-11-26T23:33:49,420 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:49,423 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411267d5827714a2942148d0a09b61ad1fb40_2b323bd2428037d76aaf73c5c85f5d46 store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:49,425 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411267d5827714a2942148d0a09b61ad1fb40_2b323bd2428037d76aaf73c5c85f5d46, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:49,425 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267d5827714a2942148d0a09b61ad1fb40_2b323bd2428037d76aaf73c5c85f5d46 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:49,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741988_1164 (size=12241) 2024-11-26T23:33:49,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741989_1165 (size=4469) 2024-11-26T23:33:49,431 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/130bdcd09139476b9600f810bf27ca5a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/130bdcd09139476b9600f810bf27ca5a 2024-11-26T23:33:49,437 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/B of 2b323bd2428037d76aaf73c5c85f5d46 into 130bdcd09139476b9600f810bf27ca5a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:49,437 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:49,437 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/B, priority=13, startTime=1732664029389; duration=0sec 2024-11-26T23:33:49,437 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:49,437 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:B 2024-11-26T23:33:49,438 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:49,439 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:49,439 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/C is initiating minor compaction (all files) 2024-11-26T23:33:49,440 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/C in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:49,440 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/0c9732d0c68b4b58a932e0a54b366a51, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/36d73214b0ba404f8e6f8d41d5d91eed, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/a69a96a4bea54d0a83fd5163801628c2] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=35.3 K 2024-11-26T23:33:49,441 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9732d0c68b4b58a932e0a54b366a51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664023089 2024-11-26T23:33:49,442 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d73214b0ba404f8e6f8d41d5d91eed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732664025276 2024-11-26T23:33:49,443 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting a69a96a4bea54d0a83fd5163801628c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732664027466 2024-11-26T23:33:49,453 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:49,454 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/b684427a0eaa4e09b737a4584df38fba is 50, key is test_row_0/C:col10/1732664028103/Put/seqid=0 2024-11-26T23:33:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741990_1166 (size=12241) 2024-11-26T23:33:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:49,650 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:49,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:49,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267688219e3b994ba59ffca9b383c31f74_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664029645/Put/seqid=0 2024-11-26T23:33:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741991_1167 (size=14744) 2024-11-26T23:33:49,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664089742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T23:33:49,812 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:49,812 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-26T23:33:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T23:33:49,815 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:49,815 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:49,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:49,832 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:49,833 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/c0d36e8968c94494976e7ba72c27f85b is 175, key is test_row_0/A:col10/1732664028103/Put/seqid=0 2024-11-26T23:33:49,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741992_1168 (size=31195) 2024-11-26T23:33:49,846 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/c0d36e8968c94494976e7ba72c27f85b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c0d36e8968c94494976e7ba72c27f85b 2024-11-26T23:33:49,853 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/A of 2b323bd2428037d76aaf73c5c85f5d46 into c0d36e8968c94494976e7ba72c27f85b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:49,853 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:49,853 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/A, priority=13, startTime=1732664029389; duration=0sec 2024-11-26T23:33:49,853 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:49,853 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:A 2024-11-26T23:33:49,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664089853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:49,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:49,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:49,874 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/b684427a0eaa4e09b737a4584df38fba as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b684427a0eaa4e09b737a4584df38fba 2024-11-26T23:33:49,885 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/C of 2b323bd2428037d76aaf73c5c85f5d46 into b684427a0eaa4e09b737a4584df38fba(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:49,885 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:49,885 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/C, priority=13, startTime=1732664029389; duration=0sec 2024-11-26T23:33:49,885 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:49,885 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:C 2024-11-26T23:33:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T23:33:49,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:49,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664090062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:50,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:50,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267688219e3b994ba59ffca9b383c31f74_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267688219e3b994ba59ffca9b383c31f74_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:50,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/c48b18ad7f6f46f2b18a97c2b3ff0d14, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:50,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/c48b18ad7f6f46f2b18a97c2b3ff0d14 is 175, key is test_row_0/A:col10/1732664029645/Put/seqid=0 2024-11-26T23:33:50,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741993_1169 (size=39699) 2024-11-26T23:33:50,083 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/c48b18ad7f6f46f2b18a97c2b3ff0d14 2024-11-26T23:33:50,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/150a97fa1b2648d4b88a8e6a04d928ae is 50, key is test_row_0/B:col10/1732664029645/Put/seqid=0 2024-11-26T23:33:50,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741994_1170 (size=12101) 2024-11-26T23:33:50,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T23:33:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664090272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664090272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664090272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:50,273 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:50,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664090272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:50,274 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:50,274 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:50,274 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:50,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664090371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:50,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:50,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:50,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T23:33:50,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/150a97fa1b2648d4b88a8e6a04d928ae 2024-11-26T23:33:50,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/18dcab60ae024964996b56ac040e7024 is 50, key is test_row_0/C:col10/1732664029645/Put/seqid=0 2024-11-26T23:33:50,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741995_1171 (size=12101) 2024-11-26T23:33:50,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:50,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:50,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,852 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T23:33:50,852 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T23:33:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664090882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:50,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:50,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:50,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/18dcab60ae024964996b56ac040e7024 2024-11-26T23:33:50,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/c48b18ad7f6f46f2b18a97c2b3ff0d14 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c48b18ad7f6f46f2b18a97c2b3ff0d14 2024-11-26T23:33:50,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c48b18ad7f6f46f2b18a97c2b3ff0d14, entries=200, sequenceid=132, filesize=38.8 K 2024-11-26T23:33:50,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/150a97fa1b2648d4b88a8e6a04d928ae as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/150a97fa1b2648d4b88a8e6a04d928ae 2024-11-26T23:33:50,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/150a97fa1b2648d4b88a8e6a04d928ae, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T23:33:50,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/18dcab60ae024964996b56ac040e7024 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/18dcab60ae024964996b56ac040e7024 2024-11-26T23:33:50,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/18dcab60ae024964996b56ac040e7024, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T23:33:50,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2b323bd2428037d76aaf73c5c85f5d46 in 1284ms, sequenceid=132, compaction requested=false 2024-11-26T23:33:50,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T23:33:51,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T23:33:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:51,043 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:33:51,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261511a156cd10449896ea51fc3d443751_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664029735/Put/seqid=0 2024-11-26T23:33:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741996_1172 (size=12304) 2024-11-26T23:33:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:51,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261511a156cd10449896ea51fc3d443751_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261511a156cd10449896ea51fc3d443751_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:51,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/5b5f8edca1074bfbbff812e5cfdaf349, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:51,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/5b5f8edca1074bfbbff812e5cfdaf349 is 175, key is test_row_0/A:col10/1732664029735/Put/seqid=0 2024-11-26T23:33:51,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741997_1173 (size=31105) 2024-11-26T23:33:51,471 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/5b5f8edca1074bfbbff812e5cfdaf349 2024-11-26T23:33:51,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b0c1e17e50c04164b54366c183556905 is 50, key is test_row_0/B:col10/1732664029735/Put/seqid=0 2024-11-26T23:33:51,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741998_1174 (size=12151) 2024-11-26T23:33:51,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b0c1e17e50c04164b54366c183556905 2024-11-26T23:33:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:51,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:51,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/5672fc64e1ac462baea90d993a557fd6 is 50, key is test_row_0/C:col10/1732664029735/Put/seqid=0 2024-11-26T23:33:51,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741999_1175 (size=12151) 2024-11-26T23:33:51,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664091919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T23:33:52,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664092032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664092241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664092282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664092282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,284 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,284 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664092291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664092291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,295 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,296 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,308 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/5672fc64e1ac462baea90d993a557fd6 2024-11-26T23:33:52,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/5b5f8edca1074bfbbff812e5cfdaf349 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/5b5f8edca1074bfbbff812e5cfdaf349 2024-11-26T23:33:52,317 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/5b5f8edca1074bfbbff812e5cfdaf349, entries=150, sequenceid=159, filesize=30.4 K 2024-11-26T23:33:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b0c1e17e50c04164b54366c183556905 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b0c1e17e50c04164b54366c183556905 2024-11-26T23:33:52,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b0c1e17e50c04164b54366c183556905, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T23:33:52,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/5672fc64e1ac462baea90d993a557fd6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5672fc64e1ac462baea90d993a557fd6 2024-11-26T23:33:52,328 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5672fc64e1ac462baea90d993a557fd6, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T23:33:52,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2b323bd2428037d76aaf73c5c85f5d46 in 1286ms, sequenceid=159, compaction requested=true 2024-11-26T23:33:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-26T23:33:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-26T23:33:52,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-26T23:33:52,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5140 sec 2024-11-26T23:33:52,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.5200 sec 2024-11-26T23:33:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:52,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T23:33:52,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264b4bed9759c54411b07910086bc2d87a_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664031918/Put/seqid=0 2024-11-26T23:33:52,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742000_1176 (size=14794) 2024-11-26T23:33:52,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664092645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664092751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664092962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:52,968 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:52,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264b4bed9759c54411b07910086bc2d87a_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264b4bed9759c54411b07910086bc2d87a_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:52,974 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/ad53d3edc0834e0c8b4dd0305c91622e, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:52,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/ad53d3edc0834e0c8b4dd0305c91622e is 175, key is test_row_0/A:col10/1732664031918/Put/seqid=0 2024-11-26T23:33:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742001_1177 (size=39749) 2024-11-26T23:33:53,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664093272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:53,379 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/ad53d3edc0834e0c8b4dd0305c91622e 2024-11-26T23:33:53,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/3fc9c44b22494f829293f6ff2d8e1830 is 50, key is test_row_0/B:col10/1732664031918/Put/seqid=0 2024-11-26T23:33:53,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742002_1178 (size=12151) 2024-11-26T23:33:53,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/3fc9c44b22494f829293f6ff2d8e1830 2024-11-26T23:33:53,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/d3444d413fcf4b6c8d435b7b66685cfa is 50, key is test_row_0/C:col10/1732664031918/Put/seqid=0 2024-11-26T23:33:53,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742003_1179 (size=12151) 2024-11-26T23:33:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664093781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:53,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/d3444d413fcf4b6c8d435b7b66685cfa 2024-11-26T23:33:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/ad53d3edc0834e0c8b4dd0305c91622e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/ad53d3edc0834e0c8b4dd0305c91622e 2024-11-26T23:33:53,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/ad53d3edc0834e0c8b4dd0305c91622e, entries=200, sequenceid=171, filesize=38.8 K 2024-11-26T23:33:53,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/3fc9c44b22494f829293f6ff2d8e1830 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3fc9c44b22494f829293f6ff2d8e1830 2024-11-26T23:33:53,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3fc9c44b22494f829293f6ff2d8e1830, entries=150, sequenceid=171, filesize=11.9 K 2024-11-26T23:33:53,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/d3444d413fcf4b6c8d435b7b66685cfa as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/d3444d413fcf4b6c8d435b7b66685cfa 2024-11-26T23:33:53,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/d3444d413fcf4b6c8d435b7b66685cfa, entries=150, sequenceid=171, filesize=11.9 K 2024-11-26T23:33:53,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2b323bd2428037d76aaf73c5c85f5d46 in 1280ms, sequenceid=171, compaction requested=true 2024-11-26T23:33:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:53,834 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:53,834 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:53,835 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:53,835 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:53,835 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/A is initiating minor compaction (all files) 2024-11-26T23:33:53,835 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/B is initiating minor compaction (all files) 2024-11-26T23:33:53,835 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/A in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:53,835 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/B in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:53,836 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c0d36e8968c94494976e7ba72c27f85b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c48b18ad7f6f46f2b18a97c2b3ff0d14, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/5b5f8edca1074bfbbff812e5cfdaf349, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/ad53d3edc0834e0c8b4dd0305c91622e] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=138.4 K 2024-11-26T23:33:53,836 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/130bdcd09139476b9600f810bf27ca5a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/150a97fa1b2648d4b88a8e6a04d928ae, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b0c1e17e50c04164b54366c183556905, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3fc9c44b22494f829293f6ff2d8e1830] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=47.5 K 2024-11-26T23:33:53,836 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:53,836 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c0d36e8968c94494976e7ba72c27f85b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c48b18ad7f6f46f2b18a97c2b3ff0d14, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/5b5f8edca1074bfbbff812e5cfdaf349, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/ad53d3edc0834e0c8b4dd0305c91622e] 2024-11-26T23:33:53,836 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 130bdcd09139476b9600f810bf27ca5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732664027466 2024-11-26T23:33:53,836 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d36e8968c94494976e7ba72c27f85b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732664027466 2024-11-26T23:33:53,837 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 150a97fa1b2648d4b88a8e6a04d928ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732664028110 2024-11-26T23:33:53,837 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting c48b18ad7f6f46f2b18a97c2b3ff0d14, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732664028110 2024-11-26T23:33:53,837 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b5f8edca1074bfbbff812e5cfdaf349, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732664029731 2024-11-26T23:33:53,837 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b0c1e17e50c04164b54366c183556905, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732664029731 2024-11-26T23:33:53,838 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc9c44b22494f829293f6ff2d8e1830, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732664031899 2024-11-26T23:33:53,838 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad53d3edc0834e0c8b4dd0305c91622e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732664031899 2024-11-26T23:33:53,860 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#B#compaction#153 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:53,861 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b90620ecbf0d4e9a8b74c984cecd9c89 is 50, key is test_row_0/B:col10/1732664031918/Put/seqid=0 2024-11-26T23:33:53,862 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:53,873 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264d55650698404242a8a0619f77b96d47_2b323bd2428037d76aaf73c5c85f5d46 store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:53,875 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264d55650698404242a8a0619f77b96d47_2b323bd2428037d76aaf73c5c85f5d46, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:53,875 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264d55650698404242a8a0619f77b96d47_2b323bd2428037d76aaf73c5c85f5d46 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:53,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742004_1180 (size=12527) 2024-11-26T23:33:53,884 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/b90620ecbf0d4e9a8b74c984cecd9c89 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b90620ecbf0d4e9a8b74c984cecd9c89 2024-11-26T23:33:53,892 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/B of 2b323bd2428037d76aaf73c5c85f5d46 into b90620ecbf0d4e9a8b74c984cecd9c89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:53,892 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:53,892 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/B, priority=12, startTime=1732664033834; duration=0sec 2024-11-26T23:33:53,892 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:53,892 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:B 2024-11-26T23:33:53,892 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:33:53,894 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:33:53,894 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/C is initiating minor compaction (all files) 2024-11-26T23:33:53,894 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/C in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:53,894 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b684427a0eaa4e09b737a4584df38fba, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/18dcab60ae024964996b56ac040e7024, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5672fc64e1ac462baea90d993a557fd6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/d3444d413fcf4b6c8d435b7b66685cfa] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=47.5 K 2024-11-26T23:33:53,895 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b684427a0eaa4e09b737a4584df38fba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732664027466 2024-11-26T23:33:53,895 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dcab60ae024964996b56ac040e7024, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732664028110 2024-11-26T23:33:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742005_1181 (size=4469) 2024-11-26T23:33:53,896 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5672fc64e1ac462baea90d993a557fd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732664029731 2024-11-26T23:33:53,896 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d3444d413fcf4b6c8d435b7b66685cfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732664031899 2024-11-26T23:33:53,908 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:53,909 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/28f471d38af94ba6992f79911d4f8e23 is 50, key is test_row_0/C:col10/1732664031918/Put/seqid=0 2024-11-26T23:33:53,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742006_1182 (size=12527) 2024-11-26T23:33:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T23:33:53,972 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:53,973 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-26T23:33:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T23:33:53,976 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:53,977 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:53,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T23:33:54,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T23:33:54,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:54,130 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:33:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112685ee80d2ca4d458798684ec614d49106_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664032643/Put/seqid=0 2024-11-26T23:33:54,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742007_1183 (size=12304) 2024-11-26T23:33:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T23:33:54,297 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:54,298 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/64e3ce1482814a67a70a6f83b7ea3d6f is 175, key is test_row_0/A:col10/1732664031918/Put/seqid=0 2024-11-26T23:33:54,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742008_1184 (size=31481) 2024-11-26T23:33:54,326 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/28f471d38af94ba6992f79911d4f8e23 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/28f471d38af94ba6992f79911d4f8e23 2024-11-26T23:33:54,332 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/C of 2b323bd2428037d76aaf73c5c85f5d46 into 28f471d38af94ba6992f79911d4f8e23(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:54,332 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:54,332 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/C, priority=12, startTime=1732664033834; duration=0sec 2024-11-26T23:33:54,333 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:54,333 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:C 2024-11-26T23:33:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:54,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112685ee80d2ca4d458798684ec614d49106_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112685ee80d2ca4d458798684ec614d49106_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:54,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/48de020f724843c79e24df49667f5d19, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:54,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/48de020f724843c79e24df49667f5d19 is 175, key is test_row_0/A:col10/1732664032643/Put/seqid=0 2024-11-26T23:33:54,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742009_1185 (size=31105) 2024-11-26T23:33:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T23:33:54,710 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/64e3ce1482814a67a70a6f83b7ea3d6f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/64e3ce1482814a67a70a6f83b7ea3d6f 2024-11-26T23:33:54,715 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/A of 2b323bd2428037d76aaf73c5c85f5d46 into 64e3ce1482814a67a70a6f83b7ea3d6f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:54,715 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:54,716 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/A, priority=12, startTime=1732664033834; duration=0sec 2024-11-26T23:33:54,716 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:54,716 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:A 2024-11-26T23:33:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:54,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664094827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664094941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:54,956 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/48de020f724843c79e24df49667f5d19 2024-11-26T23:33:54,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/c785588282ec4116909458bdba44c433 is 50, key is test_row_0/B:col10/1732664032643/Put/seqid=0 2024-11-26T23:33:54,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742010_1186 (size=12151) 2024-11-26T23:33:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T23:33:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664095151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:55,370 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/c785588282ec4116909458bdba44c433 2024-11-26T23:33:55,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/baee12852e4f408699d519c3debcc2f6 is 50, key is test_row_0/C:col10/1732664032643/Put/seqid=0 2024-11-26T23:33:55,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742011_1187 (size=12151) 2024-11-26T23:33:55,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664095461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:55,784 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/baee12852e4f408699d519c3debcc2f6 2024-11-26T23:33:55,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/48de020f724843c79e24df49667f5d19 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/48de020f724843c79e24df49667f5d19 2024-11-26T23:33:55,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/48de020f724843c79e24df49667f5d19, entries=150, sequenceid=196, filesize=30.4 K 2024-11-26T23:33:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/c785588282ec4116909458bdba44c433 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/c785588282ec4116909458bdba44c433 2024-11-26T23:33:55,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/c785588282ec4116909458bdba44c433, entries=150, sequenceid=196, filesize=11.9 K 2024-11-26T23:33:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/baee12852e4f408699d519c3debcc2f6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/baee12852e4f408699d519c3debcc2f6 2024-11-26T23:33:55,804 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/baee12852e4f408699d519c3debcc2f6, entries=150, sequenceid=196, filesize=11.9 K 2024-11-26T23:33:55,805 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2b323bd2428037d76aaf73c5c85f5d46 in 1674ms, sequenceid=196, compaction requested=false 2024-11-26T23:33:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-26T23:33:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-26T23:33:55,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-26T23:33:55,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8300 sec 2024-11-26T23:33:55,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8370 sec 2024-11-26T23:33:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:55,973 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:55,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T23:33:55,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267db426b8f3de4439b2598451af307c8c_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664035972/Put/seqid=0 2024-11-26T23:33:55,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742012_1188 (size=14794) 2024-11-26T23:33:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664096067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T23:33:56,121 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:56,122 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-26T23:33:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T23:33:56,124 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:56,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:56,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:56,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664096182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T23:33:56,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:56,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:56,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50306 deadline: 1732664096292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,297 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1732664096311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,313 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50350 deadline: 1732664096321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,322 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50364 deadline: 1732664096341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,345 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 2b323bd2428037d76aaf73c5c85f5d46 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:33:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,392 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664096391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,397 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267db426b8f3de4439b2598451af307c8c_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267db426b8f3de4439b2598451af307c8c_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:56,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/6f391b3a1b2a48e6b9379c457a11f734, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:56,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/6f391b3a1b2a48e6b9379c457a11f734 is 175, key is test_row_0/A:col10/1732664035972/Put/seqid=0 2024-11-26T23:33:56,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742013_1189 (size=39749) 2024-11-26T23:33:56,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:56,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:56,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T23:33:56,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664096701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:56,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:56,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:56,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T23:33:56,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/6f391b3a1b2a48e6b9379c457a11f734 2024-11-26T23:33:56,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/95e3b307f8b14573a68e5eae15efebda is 50, key is test_row_0/B:col10/1732664035972/Put/seqid=0 2024-11-26T23:33:56,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742014_1190 (size=12151) 2024-11-26T23:33:56,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:56,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:57,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:57,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:57,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:57,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:57,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:57,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:57,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664097211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:57,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/95e3b307f8b14573a68e5eae15efebda 2024-11-26T23:33:57,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/ffdec9f4793840149c4280102b69410c is 50, key is test_row_0/C:col10/1732664035972/Put/seqid=0 2024-11-26T23:33:57,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742015_1191 (size=12151) 2024-11-26T23:33:57,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/ffdec9f4793840149c4280102b69410c 2024-11-26T23:33:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/6f391b3a1b2a48e6b9379c457a11f734 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/6f391b3a1b2a48e6b9379c457a11f734 2024-11-26T23:33:57,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/6f391b3a1b2a48e6b9379c457a11f734, entries=200, sequenceid=211, filesize=38.8 K 2024-11-26T23:33:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/95e3b307f8b14573a68e5eae15efebda as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/95e3b307f8b14573a68e5eae15efebda 2024-11-26T23:33:57,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/95e3b307f8b14573a68e5eae15efebda, entries=150, sequenceid=211, filesize=11.9 K 2024-11-26T23:33:57,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/ffdec9f4793840149c4280102b69410c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ffdec9f4793840149c4280102b69410c 2024-11-26T23:33:57,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ffdec9f4793840149c4280102b69410c, entries=150, sequenceid=211, filesize=11.9 K 2024-11-26T23:33:57,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2b323bd2428037d76aaf73c5c85f5d46 in 1281ms, sequenceid=211, compaction requested=true 2024-11-26T23:33:57,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:57,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:57,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:57,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:57,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:57,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:57,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:57,254 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:57,254 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:57,256 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:57,256 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/A is initiating minor compaction (all files) 2024-11-26T23:33:57,256 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/A in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,256 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/64e3ce1482814a67a70a6f83b7ea3d6f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/48de020f724843c79e24df49667f5d19, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/6f391b3a1b2a48e6b9379c457a11f734] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=99.9 K 2024-11-26T23:33:57,256 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,256 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/64e3ce1482814a67a70a6f83b7ea3d6f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/48de020f724843c79e24df49667f5d19, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/6f391b3a1b2a48e6b9379c457a11f734] 2024-11-26T23:33:57,256 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:57,256 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/B is initiating minor compaction (all files) 2024-11-26T23:33:57,256 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/B in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,256 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b90620ecbf0d4e9a8b74c984cecd9c89, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/c785588282ec4116909458bdba44c433, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/95e3b307f8b14573a68e5eae15efebda] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=36.0 K 2024-11-26T23:33:57,257 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e3ce1482814a67a70a6f83b7ea3d6f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732664031899 2024-11-26T23:33:57,257 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b90620ecbf0d4e9a8b74c984cecd9c89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732664031899 2024-11-26T23:33:57,257 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48de020f724843c79e24df49667f5d19, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732664032614 2024-11-26T23:33:57,257 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c785588282ec4116909458bdba44c433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732664032614 2024-11-26T23:33:57,258 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f391b3a1b2a48e6b9379c457a11f734, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732664034811 2024-11-26T23:33:57,258 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e3b307f8b14573a68e5eae15efebda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732664034811 2024-11-26T23:33:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T23:33:57,266 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:57,270 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:57,271 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/078f41ea983e4c5c8307409328b651c8 is 50, key is test_row_0/B:col10/1732664035972/Put/seqid=0 2024-11-26T23:33:57,272 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411263221633651cd4fd8b89e8bb058abe3fb_2b323bd2428037d76aaf73c5c85f5d46 store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:57,274 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411263221633651cd4fd8b89e8bb058abe3fb_2b323bd2428037d76aaf73c5c85f5d46, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:57,274 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263221633651cd4fd8b89e8bb058abe3fb_2b323bd2428037d76aaf73c5c85f5d46 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:57,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742016_1192 (size=12629) 2024-11-26T23:33:57,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742017_1193 (size=4469) 2024-11-26T23:33:57,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T23:33:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T23:33:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112651eeec6c1e1d412e89c7ce855d1f3dd5_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664036037/Put/seqid=0 2024-11-26T23:33:57,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742018_1194 (size=12304) 2024-11-26T23:33:57,683 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:57,684 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/10b2c449d0fa49e59a5c6f67f7c2067c is 175, key is test_row_0/A:col10/1732664035972/Put/seqid=0 2024-11-26T23:33:57,691 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/078f41ea983e4c5c8307409328b651c8 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/078f41ea983e4c5c8307409328b651c8 2024-11-26T23:33:57,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742019_1195 (size=31583) 2024-11-26T23:33:57,697 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/B of 2b323bd2428037d76aaf73c5c85f5d46 into 078f41ea983e4c5c8307409328b651c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:57,697 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:57,698 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/B, priority=13, startTime=1732664037254; duration=0sec 2024-11-26T23:33:57,698 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:57,698 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:B 2024-11-26T23:33:57,698 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:57,699 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/10b2c449d0fa49e59a5c6f67f7c2067c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/10b2c449d0fa49e59a5c6f67f7c2067c 2024-11-26T23:33:57,699 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:57,699 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/C is initiating minor compaction (all files) 2024-11-26T23:33:57,700 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/C in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:57,700 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/28f471d38af94ba6992f79911d4f8e23, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/baee12852e4f408699d519c3debcc2f6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ffdec9f4793840149c4280102b69410c] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=36.0 K 2024-11-26T23:33:57,702 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 28f471d38af94ba6992f79911d4f8e23, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732664031899 2024-11-26T23:33:57,702 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting baee12852e4f408699d519c3debcc2f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732664032614 2024-11-26T23:33:57,703 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdec9f4793840149c4280102b69410c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732664034811 2024-11-26T23:33:57,706 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/A of 2b323bd2428037d76aaf73c5c85f5d46 into 10b2c449d0fa49e59a5c6f67f7c2067c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:57,706 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:57,706 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/A, priority=13, startTime=1732664037254; duration=0sec 2024-11-26T23:33:57,706 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:57,706 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:A 2024-11-26T23:33:57,710 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:57,710 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/ee052d22c87b49588e0da0465ba87130 is 50, key is test_row_0/C:col10/1732664035972/Put/seqid=0 2024-11-26T23:33:57,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742020_1196 (size=12629) 2024-11-26T23:33:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:57,774 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112651eeec6c1e1d412e89c7ce855d1f3dd5_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112651eeec6c1e1d412e89c7ce855d1f3dd5_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/adaec8bb331c49afa4b4dfca7ab44da2, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/adaec8bb331c49afa4b4dfca7ab44da2 is 175, key is test_row_0/A:col10/1732664036037/Put/seqid=0 2024-11-26T23:33:57,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742021_1197 (size=31105) 2024-11-26T23:33:57,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/adaec8bb331c49afa4b4dfca7ab44da2 2024-11-26T23:33:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/1e4716c0d4734992bf67ab10fc1a9a64 is 50, key is test_row_0/B:col10/1732664036037/Put/seqid=0 2024-11-26T23:33:57,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742022_1198 (size=12151) 2024-11-26T23:33:58,121 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/ee052d22c87b49588e0da0465ba87130 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ee052d22c87b49588e0da0465ba87130 2024-11-26T23:33:58,127 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/C of 2b323bd2428037d76aaf73c5c85f5d46 into ee052d22c87b49588e0da0465ba87130(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:33:58,127 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:58,127 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/C, priority=13, startTime=1732664037254; duration=0sec 2024-11-26T23:33:58,127 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:58,127 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:C 2024-11-26T23:33:58,191 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/1e4716c0d4734992bf67ab10fc1a9a64 2024-11-26T23:33:58,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/b26a21aa75434bd3b26f92d811b45f6c is 50, key is test_row_0/C:col10/1732664036037/Put/seqid=0 2024-11-26T23:33:58,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742023_1199 (size=12151) 2024-11-26T23:33:58,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/b26a21aa75434bd3b26f92d811b45f6c 2024-11-26T23:33:58,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/adaec8bb331c49afa4b4dfca7ab44da2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/adaec8bb331c49afa4b4dfca7ab44da2 2024-11-26T23:33:58,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/adaec8bb331c49afa4b4dfca7ab44da2, entries=150, sequenceid=234, filesize=30.4 K 2024-11-26T23:33:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/1e4716c0d4734992bf67ab10fc1a9a64 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/1e4716c0d4734992bf67ab10fc1a9a64 2024-11-26T23:33:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,219 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/1e4716c0d4734992bf67ab10fc1a9a64, entries=150, sequenceid=234, filesize=11.9 K 2024-11-26T23:33:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/b26a21aa75434bd3b26f92d811b45f6c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b26a21aa75434bd3b26f92d811b45f6c 2024-11-26T23:33:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,229 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b26a21aa75434bd3b26f92d811b45f6c, entries=150, sequenceid=234, filesize=11.9 K 2024-11-26T23:33:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,231 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 2b323bd2428037d76aaf73c5c85f5d46 in 880ms, sequenceid=234, compaction requested=false 2024-11-26T23:33:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-26T23:33:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-26T23:33:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T23:33:58,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1070 sec 2024-11-26T23:33:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.1130 sec 2024-11-26T23:33:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T23:33:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,271 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:33:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,272 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:33:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-26T23:33:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T23:33:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,274 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:33:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:33:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:33:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:33:58,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:33:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T23:33:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f7510a21a6f94ca39ce01b0818629a5f_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664038378/Put/seqid=0 2024-11-26T23:33:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742024_1200 (size=17284) 2024-11-26T23:33:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664098497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:58,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:58,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:58,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T23:33:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664098611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:58,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:58,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:58,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,797 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:33:58,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f7510a21a6f94ca39ce01b0818629a5f_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f7510a21a6f94ca39ce01b0818629a5f_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:33:58,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/35cf18aa7f824dc8a9352746e40fdf4d, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:58,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/35cf18aa7f824dc8a9352746e40fdf4d is 175, key is test_row_0/A:col10/1732664038378/Put/seqid=0 2024-11-26T23:33:58,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742025_1201 (size=48389) 2024-11-26T23:33:58,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664098822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:58,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:58,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T23:33:59,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664099131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:59,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:59,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,208 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/35cf18aa7f824dc8a9352746e40fdf4d 2024-11-26T23:33:59,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/0c461760211446c28abbb989e494049a is 50, key is test_row_0/B:col10/1732664038378/Put/seqid=0 2024-11-26T23:33:59,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742026_1202 (size=12151) 2024-11-26T23:33:59,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T23:33:59,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:59,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:59,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/0c461760211446c28abbb989e494049a 2024-11-26T23:33:59,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/5743a029cb6a43fdbd70becc5d47b167 is 50, key is test_row_0/C:col10/1732664038378/Put/seqid=0 2024-11-26T23:33:59,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742027_1203 (size=12151) 2024-11-26T23:33:59,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/5743a029cb6a43fdbd70becc5d47b167 2024-11-26T23:33:59,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/35cf18aa7f824dc8a9352746e40fdf4d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/35cf18aa7f824dc8a9352746e40fdf4d 2024-11-26T23:33:59,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:33:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664099642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:33:59,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/35cf18aa7f824dc8a9352746e40fdf4d, entries=250, sequenceid=248, filesize=47.3 K 2024-11-26T23:33:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:33:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:33:59,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/0c461760211446c28abbb989e494049a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/0c461760211446c28abbb989e494049a 2024-11-26T23:33:59,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/0c461760211446c28abbb989e494049a, entries=150, sequenceid=248, filesize=11.9 K 2024-11-26T23:33:59,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/5743a029cb6a43fdbd70becc5d47b167 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5743a029cb6a43fdbd70becc5d47b167 2024-11-26T23:33:59,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:33:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:33:59,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5743a029cb6a43fdbd70becc5d47b167, entries=150, sequenceid=248, filesize=11.9 K 2024-11-26T23:33:59,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2b323bd2428037d76aaf73c5c85f5d46 in 1277ms, sequenceid=248, compaction requested=true 2024-11-26T23:33:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:33:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:33:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:33:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:33:59,657 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:33:59,657 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:33:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:33:59,658 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:59,658 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:33:59,658 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/B is initiating minor compaction (all files) 2024-11-26T23:33:59,658 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/A is initiating minor compaction (all files) 2024-11-26T23:33:59,658 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/B in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,658 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/A in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,658 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/078f41ea983e4c5c8307409328b651c8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/1e4716c0d4734992bf67ab10fc1a9a64, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/0c461760211446c28abbb989e494049a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=36.1 K 2024-11-26T23:33:59,658 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/10b2c449d0fa49e59a5c6f67f7c2067c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/adaec8bb331c49afa4b4dfca7ab44da2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/35cf18aa7f824dc8a9352746e40fdf4d] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=108.5 K 2024-11-26T23:33:59,658 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,658 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/10b2c449d0fa49e59a5c6f67f7c2067c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/adaec8bb331c49afa4b4dfca7ab44da2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/35cf18aa7f824dc8a9352746e40fdf4d] 2024-11-26T23:33:59,659 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 078f41ea983e4c5c8307409328b651c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732664034811 2024-11-26T23:33:59,659 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b2c449d0fa49e59a5c6f67f7c2067c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732664034811 2024-11-26T23:33:59,659 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4716c0d4734992bf67ab10fc1a9a64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732664036037 2024-11-26T23:33:59,659 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting adaec8bb331c49afa4b4dfca7ab44da2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732664036037 2024-11-26T23:33:59,660 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35cf18aa7f824dc8a9352746e40fdf4d, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732664038333 2024-11-26T23:33:59,660 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c461760211446c28abbb989e494049a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732664038333 2024-11-26T23:33:59,666 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:59,669 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112627c216061f5d48f2a17bc3fcf0ec943a_2b323bd2428037d76aaf73c5c85f5d46 store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:59,669 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#B#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:59,670 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/81cf76336b0945edadfd783c7bc215ef is 50, key is test_row_0/B:col10/1732664038378/Put/seqid=0 2024-11-26T23:33:59,670 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112627c216061f5d48f2a17bc3fcf0ec943a_2b323bd2428037d76aaf73c5c85f5d46, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:59,671 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112627c216061f5d48f2a17bc3fcf0ec943a_2b323bd2428037d76aaf73c5c85f5d46 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:33:59,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742029_1205 (size=4469) 2024-11-26T23:33:59,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742028_1204 (size=12731) 2024-11-26T23:33:59,677 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#A#compaction#171 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:33:59,677 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/f536bf29d8014ad0b2ff04609449bfa4 is 175, key is test_row_0/A:col10/1732664038378/Put/seqid=0 2024-11-26T23:33:59,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742030_1206 (size=31685) 2024-11-26T23:33:59,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T23:33:59,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:33:59,808 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:33:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112604a698e679ad4e23a3a9c1c87c42ba22_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664038496/Put/seqid=0 2024-11-26T23:33:59,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742031_1207 (size=12454) 2024-11-26T23:34:00,082 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/81cf76336b0945edadfd783c7bc215ef as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/81cf76336b0945edadfd783c7bc215ef 2024-11-26T23:34:00,086 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/f536bf29d8014ad0b2ff04609449bfa4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/f536bf29d8014ad0b2ff04609449bfa4 2024-11-26T23:34:00,090 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/B of 2b323bd2428037d76aaf73c5c85f5d46 into 81cf76336b0945edadfd783c7bc215ef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:00,090 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:00,090 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/B, priority=13, startTime=1732664039657; duration=0sec 2024-11-26T23:34:00,090 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:00,090 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:B 2024-11-26T23:34:00,090 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:00,091 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/A of 2b323bd2428037d76aaf73c5c85f5d46 into f536bf29d8014ad0b2ff04609449bfa4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:00,091 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:00,091 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/A, priority=13, startTime=1732664039657; duration=0sec 2024-11-26T23:34:00,091 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:00,092 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:A 2024-11-26T23:34:00,092 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:00,092 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/C is initiating minor compaction (all files) 2024-11-26T23:34:00,092 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/C in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:00,092 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ee052d22c87b49588e0da0465ba87130, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b26a21aa75434bd3b26f92d811b45f6c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5743a029cb6a43fdbd70becc5d47b167] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=36.1 K 2024-11-26T23:34:00,092 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting ee052d22c87b49588e0da0465ba87130, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732664034811 2024-11-26T23:34:00,093 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b26a21aa75434bd3b26f92d811b45f6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732664036037 2024-11-26T23:34:00,093 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5743a029cb6a43fdbd70becc5d47b167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732664038333 2024-11-26T23:34:00,101 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:00,101 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/9d2542e17da74b698c8757e210af0c20 is 50, key is test_row_0/C:col10/1732664038378/Put/seqid=0 2024-11-26T23:34:00,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742032_1208 (size=12731) 2024-11-26T23:34:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:00,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112604a698e679ad4e23a3a9c1c87c42ba22_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112604a698e679ad4e23a3a9c1c87c42ba22_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/65790d2fb5524ce1b2d3c5b4eda0957e, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:34:00,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/65790d2fb5524ce1b2d3c5b4eda0957e is 175, key is test_row_0/A:col10/1732664038496/Put/seqid=0 2024-11-26T23:34:00,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742033_1209 (size=31255) 2024-11-26T23:34:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T23:34:00,517 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/9d2542e17da74b698c8757e210af0c20 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/9d2542e17da74b698c8757e210af0c20 2024-11-26T23:34:00,523 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/C of 2b323bd2428037d76aaf73c5c85f5d46 into 9d2542e17da74b698c8757e210af0c20(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:00,523 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:00,523 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/C, priority=13, startTime=1732664039657; duration=0sec 2024-11-26T23:34:00,523 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:00,523 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:C 2024-11-26T23:34:00,640 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/65790d2fb5524ce1b2d3c5b4eda0957e 2024-11-26T23:34:00,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/2109989122a14bc0a8c61e41bcea4315 is 50, key is test_row_0/B:col10/1732664038496/Put/seqid=0 2024-11-26T23:34:00,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742034_1210 (size=12301) 2024-11-26T23:34:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:00,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. as already flushing 2024-11-26T23:34:00,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664100679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:00,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:00,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:00,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:00,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664100791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664101001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:01,055 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/2109989122a14bc0a8c61e41bcea4315 2024-11-26T23:34:01,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/1b36617960c24b96b6b03ee3392c11d6 is 50, key is test_row_0/C:col10/1732664038496/Put/seqid=0 2024-11-26T23:34:01,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742035_1211 (size=12301) 2024-11-26T23:34:01,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1732664101311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:01,471 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/1b36617960c24b96b6b03ee3392c11d6 2024-11-26T23:34:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/65790d2fb5524ce1b2d3c5b4eda0957e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/65790d2fb5524ce1b2d3c5b4eda0957e 2024-11-26T23:34:01,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/65790d2fb5524ce1b2d3c5b4eda0957e, entries=150, sequenceid=273, filesize=30.5 K 2024-11-26T23:34:01,481 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-26T23:34:01,481 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:01,481 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,481 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:01,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/2109989122a14bc0a8c61e41bcea4315 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/2109989122a14bc0a8c61e41bcea4315 2024-11-26T23:34:01,485 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-26T23:34:01,485 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:01,485 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,485 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:01,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/2109989122a14bc0a8c61e41bcea4315, entries=150, sequenceid=273, filesize=12.0 K 2024-11-26T23:34:01,487 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-26T23:34:01,487 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:01,487 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,487 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:01,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/1b36617960c24b96b6b03ee3392c11d6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/1b36617960c24b96b6b03ee3392c11d6 2024-11-26T23:34:01,488 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-26T23:34:01,488 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:01,488 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,488 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:01,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/1b36617960c24b96b6b03ee3392c11d6, entries=150, sequenceid=273, filesize=12.0 K 2024-11-26T23:34:01,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2b323bd2428037d76aaf73c5c85f5d46 in 1685ms, sequenceid=273, compaction requested=false 2024-11-26T23:34:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-26T23:34:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-26T23:34:01,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-26T23:34:01,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2190 sec 2024-11-26T23:34:01,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.2230 sec 2024-11-26T23:34:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:01,833 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:01,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T23:34:01,833 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-26T23:34:01,833 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:01,833 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,833 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:01,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:01,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d0c0b58ba7d04b619ea8aeaf8e800f63_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_0/A:col10/1732664041831/Put/seqid=0 2024-11-26T23:34:01,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742036_1212 (size=12454) 2024-11-26T23:34:02,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:02,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d0c0b58ba7d04b619ea8aeaf8e800f63_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d0c0b58ba7d04b619ea8aeaf8e800f63_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:02,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/efbdd5292b7e4e2bbdf4b872879c0153, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:34:02,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/efbdd5292b7e4e2bbdf4b872879c0153 is 175, key is test_row_0/A:col10/1732664041831/Put/seqid=0 2024-11-26T23:34:02,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742037_1213 (size=31255) 2024-11-26T23:34:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T23:34:02,432 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:02,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/efbdd5292b7e4e2bbdf4b872879c0153 2024-11-26T23:34:02,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/a9dd9ce8db574492b30fd8dbaf343bfa is 50, key is test_row_0/B:col10/1732664041831/Put/seqid=0 2024-11-26T23:34:02,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742038_1214 (size=12301) 2024-11-26T23:34:03,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/a9dd9ce8db574492b30fd8dbaf343bfa 2024-11-26T23:34:03,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/01c7175908334767897d8f823298c131 is 50, key is test_row_0/C:col10/1732664041831/Put/seqid=0 2024-11-26T23:34:03,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742039_1215 (size=12301) 2024-11-26T23:34:03,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/01c7175908334767897d8f823298c131 2024-11-26T23:34:03,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/efbdd5292b7e4e2bbdf4b872879c0153 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/efbdd5292b7e4e2bbdf4b872879c0153 2024-11-26T23:34:03,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/efbdd5292b7e4e2bbdf4b872879c0153, entries=150, sequenceid=288, filesize=30.5 K 2024-11-26T23:34:03,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/a9dd9ce8db574492b30fd8dbaf343bfa as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/a9dd9ce8db574492b30fd8dbaf343bfa 2024-11-26T23:34:03,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/a9dd9ce8db574492b30fd8dbaf343bfa, entries=150, sequenceid=288, filesize=12.0 K 2024-11-26T23:34:03,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/01c7175908334767897d8f823298c131 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/01c7175908334767897d8f823298c131 2024-11-26T23:34:03,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/01c7175908334767897d8f823298c131, entries=150, sequenceid=288, filesize=12.0 K 2024-11-26T23:34:03,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 2b323bd2428037d76aaf73c5c85f5d46 in 1660ms, sequenceid=288, compaction requested=true 2024-11-26T23:34:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:03,493 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b323bd2428037d76aaf73c5c85f5d46:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:03,493 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:03,494 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:03,494 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:03,494 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/A is initiating minor compaction (all files) 2024-11-26T23:34:03,494 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/B is initiating minor compaction (all files) 2024-11-26T23:34:03,494 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/A in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:03,494 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/B in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:03,494 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/f536bf29d8014ad0b2ff04609449bfa4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/65790d2fb5524ce1b2d3c5b4eda0957e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/efbdd5292b7e4e2bbdf4b872879c0153] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=92.0 K 2024-11-26T23:34:03,494 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/81cf76336b0945edadfd783c7bc215ef, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/2109989122a14bc0a8c61e41bcea4315, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/a9dd9ce8db574492b30fd8dbaf343bfa] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=36.5 K 2024-11-26T23:34:03,494 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:03,494 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/f536bf29d8014ad0b2ff04609449bfa4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/65790d2fb5524ce1b2d3c5b4eda0957e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/efbdd5292b7e4e2bbdf4b872879c0153] 2024-11-26T23:34:03,494 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 81cf76336b0945edadfd783c7bc215ef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732664038333 2024-11-26T23:34:03,494 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f536bf29d8014ad0b2ff04609449bfa4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732664038333 2024-11-26T23:34:03,495 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 2109989122a14bc0a8c61e41bcea4315, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732664038439 2024-11-26T23:34:03,495 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65790d2fb5524ce1b2d3c5b4eda0957e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732664038439 2024-11-26T23:34:03,495 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting a9dd9ce8db574492b30fd8dbaf343bfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732664040665 2024-11-26T23:34:03,495 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbdd5292b7e4e2bbdf4b872879c0153, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732664040665 2024-11-26T23:34:03,501 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:34:03,503 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:03,503 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/f0e9985e4a7a4b2b8b717ebcc3266db1 is 50, key is test_row_0/B:col10/1732664041831/Put/seqid=0 2024-11-26T23:34:03,503 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126a5d77279a3ff4822b3d12071f85cb680_2b323bd2428037d76aaf73c5c85f5d46 store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:34:03,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742040_1216 (size=12983) 2024-11-26T23:34:03,521 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126a5d77279a3ff4822b3d12071f85cb680_2b323bd2428037d76aaf73c5c85f5d46, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:34:03,521 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a5d77279a3ff4822b3d12071f85cb680_2b323bd2428037d76aaf73c5c85f5d46 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:34:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742041_1217 (size=4469) 2024-11-26T23:34:03,913 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/f0e9985e4a7a4b2b8b717ebcc3266db1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/f0e9985e4a7a4b2b8b717ebcc3266db1 2024-11-26T23:34:03,917 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/B of 2b323bd2428037d76aaf73c5c85f5d46 into f0e9985e4a7a4b2b8b717ebcc3266db1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:03,917 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:03,917 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/B, priority=13, startTime=1732664043493; duration=0sec 2024-11-26T23:34:03,917 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:03,917 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:B 2024-11-26T23:34:03,917 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:03,918 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:03,918 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 2b323bd2428037d76aaf73c5c85f5d46/C is initiating minor compaction (all files) 2024-11-26T23:34:03,918 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b323bd2428037d76aaf73c5c85f5d46/C in TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:03,918 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/9d2542e17da74b698c8757e210af0c20, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/1b36617960c24b96b6b03ee3392c11d6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/01c7175908334767897d8f823298c131] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp, totalSize=36.5 K 2024-11-26T23:34:03,919 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2542e17da74b698c8757e210af0c20, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732664038333 2024-11-26T23:34:03,919 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b36617960c24b96b6b03ee3392c11d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732664038439 2024-11-26T23:34:03,919 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c7175908334767897d8f823298c131, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732664040665 2024-11-26T23:34:03,926 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:03,927 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b323bd2428037d76aaf73c5c85f5d46#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:03,927 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/05a84dd8a722404f9f1b1c25e62c0180 is 175, key is test_row_0/A:col10/1732664041831/Put/seqid=0 2024-11-26T23:34:03,927 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/2ebf8d02897942c583594e4822bd2ee3 is 50, key is test_row_0/C:col10/1732664041831/Put/seqid=0 2024-11-26T23:34:03,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742043_1219 (size=12983) 2024-11-26T23:34:03,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742042_1218 (size=31937) 2024-11-26T23:34:04,336 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/05a84dd8a722404f9f1b1c25e62c0180 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/05a84dd8a722404f9f1b1c25e62c0180 2024-11-26T23:34:04,336 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/2ebf8d02897942c583594e4822bd2ee3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2ebf8d02897942c583594e4822bd2ee3 2024-11-26T23:34:04,341 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/C of 2b323bd2428037d76aaf73c5c85f5d46 into 2ebf8d02897942c583594e4822bd2ee3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:04,341 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:04,341 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/C, priority=13, startTime=1732664043493; duration=0sec 2024-11-26T23:34:04,341 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b323bd2428037d76aaf73c5c85f5d46/A of 2b323bd2428037d76aaf73c5c85f5d46 into 05a84dd8a722404f9f1b1c25e62c0180(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:04,341 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:04,341 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:C 2024-11-26T23:34:04,341 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b323bd2428037d76aaf73c5c85f5d46: 2024-11-26T23:34:04,341 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., storeName=2b323bd2428037d76aaf73c5c85f5d46/A, priority=13, startTime=1732664043493; duration=0sec 2024-11-26T23:34:04,341 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:04,341 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b323bd2428037d76aaf73c5c85f5d46:A 2024-11-26T23:34:04,597 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:34:06,362 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-26T23:34:06,362 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:06,362 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,363 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:06,382 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-26T23:34:06,383 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:06,383 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,383 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:06,393 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-26T23:34:06,393 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:06,393 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,393 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:06,413 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-11-26T23:34:06,413 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:06,413 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,413 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:06,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T23:34:06,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-26T23:34:06,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-26T23:34:06,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-26T23:34:06,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-26T23:34:06,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-26T23:34:06,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T23:34:06,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4016 2024-11-26T23:34:06,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3823 2024-11-26T23:34:06,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T23:34:06,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1641 2024-11-26T23:34:06,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4920 rows 2024-11-26T23:34:06,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1642 2024-11-26T23:34:06,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4925 rows 2024-11-26T23:34:06,414 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:34:06,414 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:34:06,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:06,414 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:06,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T23:34:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T23:34:06,417 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664046417"}]},"ts":"1732664046417"} 2024-11-26T23:34:06,419 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T23:34:06,419 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T23:34:06,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:34:06,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, UNASSIGN}] 2024-11-26T23:34:06,421 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, UNASSIGN 2024-11-26T23:34:06,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:06,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, UNASSIGN because future has completed 2024-11-26T23:34:06,423 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:34:06,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:34:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T23:34:06,575 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:06,575 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:34:06,575 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 2b323bd2428037d76aaf73c5c85f5d46, disabling compactions & flushes 2024-11-26T23:34:06,575 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:06,575 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:06,575 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. after waiting 0 ms 2024-11-26T23:34:06,575 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:06,575 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 2b323bd2428037d76aaf73c5c85f5d46 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-26T23:34:06,581 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112687161e5745344b0c8ceaa95257661c4f_2b323bd2428037d76aaf73c5c85f5d46 is 50, key is test_row_1/A:col10/1732664046392/Put/seqid=0 2024-11-26T23:34:06,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742044_1220 (size=9914) 2024-11-26T23:34:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T23:34:06,985 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:06,990 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112687161e5745344b0c8ceaa95257661c4f_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112687161e5745344b0c8ceaa95257661c4f_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:06,990 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/8b15a6e24a2f4ec0a8f7f9543df12807, store: [table=TestAcidGuarantees family=A region=2b323bd2428037d76aaf73c5c85f5d46] 2024-11-26T23:34:06,991 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/8b15a6e24a2f4ec0a8f7f9543df12807 is 175, key is test_row_1/A:col10/1732664046392/Put/seqid=0 2024-11-26T23:34:06,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742045_1221 (size=22561) 2024-11-26T23:34:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T23:34:07,395 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/8b15a6e24a2f4ec0a8f7f9543df12807 2024-11-26T23:34:07,402 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/3f70bcdcdc2748ad88c58c71bc29a392 is 50, key is test_row_1/B:col10/1732664046392/Put/seqid=0 2024-11-26T23:34:07,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742046_1222 (size=9857) 2024-11-26T23:34:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T23:34:07,806 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/3f70bcdcdc2748ad88c58c71bc29a392 2024-11-26T23:34:07,813 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/226ebd782bba40989eb9493e323f3aef is 50, key is test_row_1/C:col10/1732664046392/Put/seqid=0 2024-11-26T23:34:07,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742047_1223 (size=9857) 2024-11-26T23:34:08,217 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/226ebd782bba40989eb9493e323f3aef 2024-11-26T23:34:08,222 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/A/8b15a6e24a2f4ec0a8f7f9543df12807 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/8b15a6e24a2f4ec0a8f7f9543df12807 2024-11-26T23:34:08,225 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/8b15a6e24a2f4ec0a8f7f9543df12807, entries=100, sequenceid=298, filesize=22.0 K 2024-11-26T23:34:08,226 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/B/3f70bcdcdc2748ad88c58c71bc29a392 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3f70bcdcdc2748ad88c58c71bc29a392 2024-11-26T23:34:08,229 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3f70bcdcdc2748ad88c58c71bc29a392, entries=100, sequenceid=298, filesize=9.6 K 2024-11-26T23:34:08,230 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/.tmp/C/226ebd782bba40989eb9493e323f3aef as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/226ebd782bba40989eb9493e323f3aef 2024-11-26T23:34:08,233 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/226ebd782bba40989eb9493e323f3aef, entries=100, sequenceid=298, filesize=9.6 K 2024-11-26T23:34:08,234 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2b323bd2428037d76aaf73c5c85f5d46 in 1659ms, sequenceid=298, compaction requested=false 2024-11-26T23:34:08,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/915e9de783f341f99eb6e8eb68c95308, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/0f41af5ba87041ba9eddbe552d5faf48, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/3ff6754e165e4192a26743a00e8ffca8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/e9c40f34bcd14e32bd25ca6c0795a84f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/55195552f75e46a9bbbb228deeb741c5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/d2d1db480929488eab9f463eac67b97e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/caa8474b01564ebe8e60573e1a8de657, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c0d36e8968c94494976e7ba72c27f85b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c48b18ad7f6f46f2b18a97c2b3ff0d14, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/5b5f8edca1074bfbbff812e5cfdaf349, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/ad53d3edc0834e0c8b4dd0305c91622e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/64e3ce1482814a67a70a6f83b7ea3d6f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/48de020f724843c79e24df49667f5d19, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/6f391b3a1b2a48e6b9379c457a11f734, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/10b2c449d0fa49e59a5c6f67f7c2067c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/adaec8bb331c49afa4b4dfca7ab44da2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/35cf18aa7f824dc8a9352746e40fdf4d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/f536bf29d8014ad0b2ff04609449bfa4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/65790d2fb5524ce1b2d3c5b4eda0957e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/efbdd5292b7e4e2bbdf4b872879c0153] to archive 2024-11-26T23:34:08,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:34:08,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/915e9de783f341f99eb6e8eb68c95308 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/915e9de783f341f99eb6e8eb68c95308 2024-11-26T23:34:08,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/0f41af5ba87041ba9eddbe552d5faf48 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/0f41af5ba87041ba9eddbe552d5faf48 2024-11-26T23:34:08,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/3ff6754e165e4192a26743a00e8ffca8 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/3ff6754e165e4192a26743a00e8ffca8 2024-11-26T23:34:08,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/e9c40f34bcd14e32bd25ca6c0795a84f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/e9c40f34bcd14e32bd25ca6c0795a84f 2024-11-26T23:34:08,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/55195552f75e46a9bbbb228deeb741c5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/55195552f75e46a9bbbb228deeb741c5 2024-11-26T23:34:08,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/d2d1db480929488eab9f463eac67b97e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/d2d1db480929488eab9f463eac67b97e 2024-11-26T23:34:08,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/caa8474b01564ebe8e60573e1a8de657 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/caa8474b01564ebe8e60573e1a8de657 2024-11-26T23:34:08,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c0d36e8968c94494976e7ba72c27f85b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c0d36e8968c94494976e7ba72c27f85b 2024-11-26T23:34:08,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c48b18ad7f6f46f2b18a97c2b3ff0d14 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/c48b18ad7f6f46f2b18a97c2b3ff0d14 2024-11-26T23:34:08,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/5b5f8edca1074bfbbff812e5cfdaf349 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/5b5f8edca1074bfbbff812e5cfdaf349 2024-11-26T23:34:08,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/ad53d3edc0834e0c8b4dd0305c91622e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/ad53d3edc0834e0c8b4dd0305c91622e 2024-11-26T23:34:08,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/64e3ce1482814a67a70a6f83b7ea3d6f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/64e3ce1482814a67a70a6f83b7ea3d6f 2024-11-26T23:34:08,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/48de020f724843c79e24df49667f5d19 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/48de020f724843c79e24df49667f5d19 2024-11-26T23:34:08,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/6f391b3a1b2a48e6b9379c457a11f734 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/6f391b3a1b2a48e6b9379c457a11f734 2024-11-26T23:34:08,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/10b2c449d0fa49e59a5c6f67f7c2067c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/10b2c449d0fa49e59a5c6f67f7c2067c 2024-11-26T23:34:08,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/adaec8bb331c49afa4b4dfca7ab44da2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/adaec8bb331c49afa4b4dfca7ab44da2 2024-11-26T23:34:08,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/35cf18aa7f824dc8a9352746e40fdf4d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/35cf18aa7f824dc8a9352746e40fdf4d 2024-11-26T23:34:08,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/f536bf29d8014ad0b2ff04609449bfa4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/f536bf29d8014ad0b2ff04609449bfa4 2024-11-26T23:34:08,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/65790d2fb5524ce1b2d3c5b4eda0957e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/65790d2fb5524ce1b2d3c5b4eda0957e 2024-11-26T23:34:08,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/efbdd5292b7e4e2bbdf4b872879c0153 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/efbdd5292b7e4e2bbdf4b872879c0153 2024-11-26T23:34:08,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/8c7ee126b092434faa8fea9d4f8cf402, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/74edd45ea1084816bea619d2a3195345, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/71ad390683e94543b24ac3fa36d1c2cf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/6974491dba1d49ceb32fbb971a08ca35, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b8641d866723435aa682a9419a487ea5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/130bdcd09139476b9600f810bf27ca5a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/fff4cdfe68ee45c1a5713596bbf2d332, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/150a97fa1b2648d4b88a8e6a04d928ae, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b0c1e17e50c04164b54366c183556905, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b90620ecbf0d4e9a8b74c984cecd9c89, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3fc9c44b22494f829293f6ff2d8e1830, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/c785588282ec4116909458bdba44c433, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/078f41ea983e4c5c8307409328b651c8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/95e3b307f8b14573a68e5eae15efebda, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/1e4716c0d4734992bf67ab10fc1a9a64, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/81cf76336b0945edadfd783c7bc215ef, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/0c461760211446c28abbb989e494049a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/2109989122a14bc0a8c61e41bcea4315, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/a9dd9ce8db574492b30fd8dbaf343bfa] to archive 2024-11-26T23:34:08,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:34:08,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/8c7ee126b092434faa8fea9d4f8cf402 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/8c7ee126b092434faa8fea9d4f8cf402 2024-11-26T23:34:08,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/74edd45ea1084816bea619d2a3195345 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/74edd45ea1084816bea619d2a3195345 2024-11-26T23:34:08,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/71ad390683e94543b24ac3fa36d1c2cf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/71ad390683e94543b24ac3fa36d1c2cf 2024-11-26T23:34:08,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/6974491dba1d49ceb32fbb971a08ca35 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/6974491dba1d49ceb32fbb971a08ca35 2024-11-26T23:34:08,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/ebb93cfdd4ae4c85ba13a0d5b3bec3d0 2024-11-26T23:34:08,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b8641d866723435aa682a9419a487ea5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b8641d866723435aa682a9419a487ea5 2024-11-26T23:34:08,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/130bdcd09139476b9600f810bf27ca5a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/130bdcd09139476b9600f810bf27ca5a 2024-11-26T23:34:08,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/fff4cdfe68ee45c1a5713596bbf2d332 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/fff4cdfe68ee45c1a5713596bbf2d332 2024-11-26T23:34:08,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/150a97fa1b2648d4b88a8e6a04d928ae to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/150a97fa1b2648d4b88a8e6a04d928ae 2024-11-26T23:34:08,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b0c1e17e50c04164b54366c183556905 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b0c1e17e50c04164b54366c183556905 2024-11-26T23:34:08,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b90620ecbf0d4e9a8b74c984cecd9c89 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/b90620ecbf0d4e9a8b74c984cecd9c89 2024-11-26T23:34:08,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3fc9c44b22494f829293f6ff2d8e1830 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3fc9c44b22494f829293f6ff2d8e1830 2024-11-26T23:34:08,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/c785588282ec4116909458bdba44c433 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/c785588282ec4116909458bdba44c433 2024-11-26T23:34:08,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/078f41ea983e4c5c8307409328b651c8 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/078f41ea983e4c5c8307409328b651c8 2024-11-26T23:34:08,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/95e3b307f8b14573a68e5eae15efebda to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/95e3b307f8b14573a68e5eae15efebda 2024-11-26T23:34:08,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/1e4716c0d4734992bf67ab10fc1a9a64 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/1e4716c0d4734992bf67ab10fc1a9a64 2024-11-26T23:34:08,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/81cf76336b0945edadfd783c7bc215ef to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/81cf76336b0945edadfd783c7bc215ef 2024-11-26T23:34:08,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/0c461760211446c28abbb989e494049a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/0c461760211446c28abbb989e494049a 2024-11-26T23:34:08,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/2109989122a14bc0a8c61e41bcea4315 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/2109989122a14bc0a8c61e41bcea4315 2024-11-26T23:34:08,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/a9dd9ce8db574492b30fd8dbaf343bfa to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/a9dd9ce8db574492b30fd8dbaf343bfa 2024-11-26T23:34:08,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/62bef4aa710c4702a9f411a149f3b829, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/fe28f94dc0ba4e899debecd0a4487a6b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2bbff5fda9ae4a44beb240a7f594e40e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/0c9732d0c68b4b58a932e0a54b366a51, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/c702e81503af4f7cbc806636b3a0d574, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/36d73214b0ba404f8e6f8d41d5d91eed, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b684427a0eaa4e09b737a4584df38fba, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/a69a96a4bea54d0a83fd5163801628c2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/18dcab60ae024964996b56ac040e7024, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5672fc64e1ac462baea90d993a557fd6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/28f471d38af94ba6992f79911d4f8e23, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/d3444d413fcf4b6c8d435b7b66685cfa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/baee12852e4f408699d519c3debcc2f6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ee052d22c87b49588e0da0465ba87130, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ffdec9f4793840149c4280102b69410c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b26a21aa75434bd3b26f92d811b45f6c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/9d2542e17da74b698c8757e210af0c20, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5743a029cb6a43fdbd70becc5d47b167, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/1b36617960c24b96b6b03ee3392c11d6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/01c7175908334767897d8f823298c131] to archive 2024-11-26T23:34:08,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:34:08,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/62bef4aa710c4702a9f411a149f3b829 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/62bef4aa710c4702a9f411a149f3b829 2024-11-26T23:34:08,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/fe28f94dc0ba4e899debecd0a4487a6b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/fe28f94dc0ba4e899debecd0a4487a6b 2024-11-26T23:34:08,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2bbff5fda9ae4a44beb240a7f594e40e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2bbff5fda9ae4a44beb240a7f594e40e 2024-11-26T23:34:08,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/0c9732d0c68b4b58a932e0a54b366a51 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/0c9732d0c68b4b58a932e0a54b366a51 2024-11-26T23:34:08,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/c702e81503af4f7cbc806636b3a0d574 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/c702e81503af4f7cbc806636b3a0d574 2024-11-26T23:34:08,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/36d73214b0ba404f8e6f8d41d5d91eed to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/36d73214b0ba404f8e6f8d41d5d91eed 2024-11-26T23:34:08,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b684427a0eaa4e09b737a4584df38fba to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b684427a0eaa4e09b737a4584df38fba 2024-11-26T23:34:08,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/a69a96a4bea54d0a83fd5163801628c2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/a69a96a4bea54d0a83fd5163801628c2 2024-11-26T23:34:08,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/18dcab60ae024964996b56ac040e7024 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/18dcab60ae024964996b56ac040e7024 2024-11-26T23:34:08,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5672fc64e1ac462baea90d993a557fd6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5672fc64e1ac462baea90d993a557fd6 2024-11-26T23:34:08,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/28f471d38af94ba6992f79911d4f8e23 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/28f471d38af94ba6992f79911d4f8e23 2024-11-26T23:34:08,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/d3444d413fcf4b6c8d435b7b66685cfa to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/d3444d413fcf4b6c8d435b7b66685cfa 2024-11-26T23:34:08,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/baee12852e4f408699d519c3debcc2f6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/baee12852e4f408699d519c3debcc2f6 2024-11-26T23:34:08,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ee052d22c87b49588e0da0465ba87130 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ee052d22c87b49588e0da0465ba87130 2024-11-26T23:34:08,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ffdec9f4793840149c4280102b69410c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/ffdec9f4793840149c4280102b69410c 2024-11-26T23:34:08,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b26a21aa75434bd3b26f92d811b45f6c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/b26a21aa75434bd3b26f92d811b45f6c 2024-11-26T23:34:08,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/9d2542e17da74b698c8757e210af0c20 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/9d2542e17da74b698c8757e210af0c20 2024-11-26T23:34:08,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5743a029cb6a43fdbd70becc5d47b167 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/5743a029cb6a43fdbd70becc5d47b167 2024-11-26T23:34:08,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/1b36617960c24b96b6b03ee3392c11d6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/1b36617960c24b96b6b03ee3392c11d6 2024-11-26T23:34:08,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/01c7175908334767897d8f823298c131 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/01c7175908334767897d8f823298c131 2024-11-26T23:34:08,299 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/recovered.edits/301.seqid, newMaxSeqId=301, maxSeqId=4 2024-11-26T23:34:08,300 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. 2024-11-26T23:34:08,300 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 2b323bd2428037d76aaf73c5c85f5d46: Waiting for close lock at 1732664046575Running coprocessor pre-close hooks at 1732664046575Disabling compacts and flushes for region at 1732664046575Disabling writes for close at 1732664046575Obtaining lock to block concurrent updates at 1732664046575Preparing flush snapshotting stores in 2b323bd2428037d76aaf73c5c85f5d46 at 1732664046575Finished memstore snapshotting TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732664046576 (+1 ms)Flushing stores of TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46. at 1732664046576Flushing 2b323bd2428037d76aaf73c5c85f5d46/A: creating writer at 1732664046576Flushing mob file 2b323bd2428037d76aaf73c5c85f5d46/A: appending metadata at 1732664046581 (+5 ms)Flushing mob file 2b323bd2428037d76aaf73c5c85f5d46/A: closing flushed file at 1732664046581Flushing 2b323bd2428037d76aaf73c5c85f5d46/A: appending metadata at 1732664046990 (+409 ms)Flushing 2b323bd2428037d76aaf73c5c85f5d46/A: closing flushed file at 1732664046991 (+1 ms)Flushing 2b323bd2428037d76aaf73c5c85f5d46/B: creating writer at 1732664047399 (+408 ms)Flushing 2b323bd2428037d76aaf73c5c85f5d46/B: appending metadata at 1732664047402 (+3 ms)Flushing 2b323bd2428037d76aaf73c5c85f5d46/B: closing flushed file at 1732664047402Flushing 2b323bd2428037d76aaf73c5c85f5d46/C: creating writer at 1732664047810 (+408 ms)Flushing 2b323bd2428037d76aaf73c5c85f5d46/C: appending metadata at 1732664047813 (+3 ms)Flushing 2b323bd2428037d76aaf73c5c85f5d46/C: closing flushed file at 1732664047813Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3442c64b: reopening flushed file at 1732664048221 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c8ccee2: reopening flushed file at 1732664048225 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689aa851: reopening flushed file at 1732664048229 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2b323bd2428037d76aaf73c5c85f5d46 in 1659ms, sequenceid=298, compaction requested=false at 1732664048234 (+5 ms)Writing region close event to WAL at 1732664048296 (+62 ms)Running coprocessor post-close hooks at 1732664048300 (+4 ms)Closed at 1732664048300 2024-11-26T23:34:08,301 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,302 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=2b323bd2428037d76aaf73c5c85f5d46, regionState=CLOSED 2024-11-26T23:34:08,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:34:08,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-26T23:34:08,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 2b323bd2428037d76aaf73c5c85f5d46, server=c707e9ace675,43077,1732663986534 in 1.8810 sec 2024-11-26T23:34:08,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-26T23:34:08,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b323bd2428037d76aaf73c5c85f5d46, UNASSIGN in 1.8850 sec 2024-11-26T23:34:08,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-26T23:34:08,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8880 sec 2024-11-26T23:34:08,310 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664048310"}]},"ts":"1732664048310"} 2024-11-26T23:34:08,311 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T23:34:08,311 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T23:34:08,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8970 sec 2024-11-26T23:34:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T23:34:08,562 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:08,562 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T23:34:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:08,564 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T23:34:08,564 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:08,567 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,569 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/recovered.edits] 2024-11-26T23:34:08,572 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/05a84dd8a722404f9f1b1c25e62c0180 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/05a84dd8a722404f9f1b1c25e62c0180 2024-11-26T23:34:08,573 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/8b15a6e24a2f4ec0a8f7f9543df12807 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/A/8b15a6e24a2f4ec0a8f7f9543df12807 2024-11-26T23:34:08,575 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3f70bcdcdc2748ad88c58c71bc29a392 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/3f70bcdcdc2748ad88c58c71bc29a392 2024-11-26T23:34:08,577 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/f0e9985e4a7a4b2b8b717ebcc3266db1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/B/f0e9985e4a7a4b2b8b717ebcc3266db1 2024-11-26T23:34:08,579 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/226ebd782bba40989eb9493e323f3aef to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/226ebd782bba40989eb9493e323f3aef 2024-11-26T23:34:08,580 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2ebf8d02897942c583594e4822bd2ee3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/C/2ebf8d02897942c583594e4822bd2ee3 2024-11-26T23:34:08,582 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/recovered.edits/301.seqid to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46/recovered.edits/301.seqid 2024-11-26T23:34:08,583 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,583 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T23:34:08,583 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T23:34:08,584 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T23:34:08,587 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112604a698e679ad4e23a3a9c1c87c42ba22_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112604a698e679ad4e23a3a9c1c87c42ba22_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,589 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261511a156cd10449896ea51fc3d443751_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261511a156cd10449896ea51fc3d443751_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,590 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262ce790652f5242369eadcb9941c99375_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262ce790652f5242369eadcb9941c99375_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,591 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263fe889d15dec489b8f972e363443fa41_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263fe889d15dec489b8f972e363443fa41_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,592 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264b4bed9759c54411b07910086bc2d87a_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264b4bed9759c54411b07910086bc2d87a_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,593 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112651eeec6c1e1d412e89c7ce855d1f3dd5_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112651eeec6c1e1d412e89c7ce855d1f3dd5_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,594 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267688219e3b994ba59ffca9b383c31f74_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267688219e3b994ba59ffca9b383c31f74_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,595 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267db426b8f3de4439b2598451af307c8c_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267db426b8f3de4439b2598451af307c8c_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,596 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112685ee80d2ca4d458798684ec614d49106_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112685ee80d2ca4d458798684ec614d49106_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,597 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112687161e5745344b0c8ceaa95257661c4f_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112687161e5745344b0c8ceaa95257661c4f_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,598 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268895761d238d4f1c9ebcaec18b407d63_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268895761d238d4f1c9ebcaec18b407d63_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,599 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8a7a1e6e8d24b29b2425a6a7579b33c_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8a7a1e6e8d24b29b2425a6a7579b33c_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,600 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d0c0b58ba7d04b619ea8aeaf8e800f63_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d0c0b58ba7d04b619ea8aeaf8e800f63_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,601 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e2f19a735f8c4502a767481cf30b515c_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e2f19a735f8c4502a767481cf30b515c_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,602 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec1c471fd1854737b4d87b09a7f66984_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec1c471fd1854737b4d87b09a7f66984_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,603 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f7510a21a6f94ca39ce01b0818629a5f_2b323bd2428037d76aaf73c5c85f5d46 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f7510a21a6f94ca39ce01b0818629a5f_2b323bd2428037d76aaf73c5c85f5d46 2024-11-26T23:34:08,604 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T23:34:08,606 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:08,609 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T23:34:08,612 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T23:34:08,613 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:08,613 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T23:34:08,613 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732664048613"}]},"ts":"9223372036854775807"} 2024-11-26T23:34:08,615 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T23:34:08,615 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2b323bd2428037d76aaf73c5c85f5d46, NAME => 'TestAcidGuarantees,,1732664018333.2b323bd2428037d76aaf73c5c85f5d46.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T23:34:08,615 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T23:34:08,615 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732664048615"}]},"ts":"9223372036854775807"} 2024-11-26T23:34:08,617 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T23:34:08,618 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:08,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-26T23:34:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T23:34:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T23:34:08,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:08,681 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 239), OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 353), ProcessCount=11 (was 11), AvailableMemoryMB=6733 (was 6823) 2024-11-26T23:34:08,689 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=6733 2024-11-26T23:34:08,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:34:08,690 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:34:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:08,692 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:34:08,692 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:08,692 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-26T23:34:08,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-26T23:34:08,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:34:08,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742048_1224 (size=959) 2024-11-26T23:34:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-26T23:34:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-26T23:34:09,100 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:34:09,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742049_1225 (size=53) 2024-11-26T23:34:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-26T23:34:09,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:34:09,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c5301f81964215ffd1194b5e2f86d347, disabling compactions & flushes 2024-11-26T23:34:09,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:09,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:09,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. after waiting 0 ms 2024-11-26T23:34:09,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:09,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:09,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c5301f81964215ffd1194b5e2f86d347: Waiting for close lock at 1732664049520Disabling compacts and flushes for region at 1732664049520Disabling writes for close at 1732664049520Writing region close event to WAL at 1732664049520Closed at 1732664049520 2024-11-26T23:34:09,521 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:34:09,522 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732664049521"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732664049521"}]},"ts":"1732664049521"} 2024-11-26T23:34:09,523 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:34:09,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:34:09,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664049523"}]},"ts":"1732664049523"} 2024-11-26T23:34:09,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T23:34:09,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, ASSIGN}] 2024-11-26T23:34:09,526 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, ASSIGN 2024-11-26T23:34:09,526 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, ASSIGN; state=OFFLINE, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=false 2024-11-26T23:34:09,677 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=c5301f81964215ffd1194b5e2f86d347, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:09,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, ASSIGN because future has completed 2024-11-26T23:34:09,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:34:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-26T23:34:09,833 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:09,833 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:34:09,834 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,834 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:34:09,834 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,834 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,835 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,836 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5301f81964215ffd1194b5e2f86d347 columnFamilyName A 2024-11-26T23:34:09,836 DEBUG [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:09,837 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] regionserver.HStore(327): Store=c5301f81964215ffd1194b5e2f86d347/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:09,837 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,838 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5301f81964215ffd1194b5e2f86d347 columnFamilyName B 2024-11-26T23:34:09,838 DEBUG [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:09,838 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] regionserver.HStore(327): Store=c5301f81964215ffd1194b5e2f86d347/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:09,839 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,839 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5301f81964215ffd1194b5e2f86d347 columnFamilyName C 2024-11-26T23:34:09,839 DEBUG [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:09,840 INFO [StoreOpener-c5301f81964215ffd1194b5e2f86d347-1 {}] regionserver.HStore(327): Store=c5301f81964215ffd1194b5e2f86d347/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:09,840 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,841 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,841 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,842 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,842 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,842 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:34:09,843 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,845 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:34:09,845 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened c5301f81964215ffd1194b5e2f86d347; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75365619, jitterRate=0.1230352371931076}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:34:09,845 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:09,846 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for c5301f81964215ffd1194b5e2f86d347: Running coprocessor pre-open hook at 1732664049834Writing region info on filesystem at 1732664049834Initializing all the Stores at 1732664049834Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664049835 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664049835Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664049835Cleaning up temporary data from old regions at 1732664049842 (+7 ms)Running coprocessor post-open hooks at 1732664049845 (+3 ms)Region opened successfully at 1732664049846 (+1 ms) 2024-11-26T23:34:09,846 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., pid=61, masterSystemTime=1732664049830 2024-11-26T23:34:09,848 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:09,848 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:09,848 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=c5301f81964215ffd1194b5e2f86d347, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:09,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:34:09,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T23:34:09,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 in 172 msec 2024-11-26T23:34:09,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-26T23:34:09,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, ASSIGN in 327 msec 2024-11-26T23:34:09,854 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:34:09,855 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664049854"}]},"ts":"1732664049854"} 2024-11-26T23:34:09,856 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T23:34:09,856 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:34:09,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-26T23:34:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-26T23:34:10,841 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:10,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd12d34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,846 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6dd23f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,847 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,850 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:34:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35833,1732663985624 2024-11-26T23:34:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3cadb4eb 2024-11-26T23:34:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:34:10,851 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:34:10,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,853 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755ae001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,855 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456cc7e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,856 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7388b2ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,857 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48312d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,859 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e02caee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,860 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a74370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,862 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf5958e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,863 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796b6036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,865 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76148c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4862bd6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,869 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c502285, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,870 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4decd049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,872 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708a58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,873 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3917a01b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,875 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb4c1f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,876 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e99969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,877 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e5b7a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21de45c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1d0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:10,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:10,880 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:10,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:10,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:10,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71027090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:10,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411037af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:10,882 DEBUG [Thread-1064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,882 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,882 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,882 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,882 DEBUG [Thread-1066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,882 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,883 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,883 DEBUG [Thread-1066 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,883 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,884 DEBUG [Thread-1066 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,884 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,885 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:10,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-26T23:34:10,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T23:34:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,893 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:10,893 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:10,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:34:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:10,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:10,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:10,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/4ddb75a354124179a30fae6c19a41bdc is 50, key is test_row_0/A:col10/1732664050897/Put/seqid=0 2024-11-26T23:34:10,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664110935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:10,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664110936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664110938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664110940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664110941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:10,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742050_1226 (size=14341) 2024-11-26T23:34:10,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/4ddb75a354124179a30fae6c19a41bdc 2024-11-26T23:34:10,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/0496a22283064e229ab1e49a199bcdaf is 50, key is test_row_0/B:col10/1732664050897/Put/seqid=0 2024-11-26T23:34:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T23:34:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742051_1227 (size=12001) 2024-11-26T23:34:11,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/0496a22283064e229ab1e49a199bcdaf 2024-11-26T23:34:11,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T23:34:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:11,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:11,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:11,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/cf386b5fa760412aaffa14dd4eed42a7 is 50, key is test_row_0/C:col10/1732664050897/Put/seqid=0 2024-11-26T23:34:11,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664111052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664111052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664111053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664111053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664111053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742052_1228 (size=12001) 2024-11-26T23:34:11,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/cf386b5fa760412aaffa14dd4eed42a7 2024-11-26T23:34:11,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/4ddb75a354124179a30fae6c19a41bdc as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/4ddb75a354124179a30fae6c19a41bdc 2024-11-26T23:34:11,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/4ddb75a354124179a30fae6c19a41bdc, entries=200, sequenceid=13, filesize=14.0 K 2024-11-26T23:34:11,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/0496a22283064e229ab1e49a199bcdaf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/0496a22283064e229ab1e49a199bcdaf 2024-11-26T23:34:11,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/0496a22283064e229ab1e49a199bcdaf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:34:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/cf386b5fa760412aaffa14dd4eed42a7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/cf386b5fa760412aaffa14dd4eed42a7 2024-11-26T23:34:11,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/cf386b5fa760412aaffa14dd4eed42a7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:34:11,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c5301f81964215ffd1194b5e2f86d347 in 195ms, sequenceid=13, compaction requested=false 2024-11-26T23:34:11,092 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T23:34:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:11,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T23:34:11,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:11,202 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:34:11,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/a8fd74c10886444db53616a44fc39259 is 50, key is test_row_0/A:col10/1732664050938/Put/seqid=0 2024-11-26T23:34:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T23:34:11,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742053_1229 (size=12001) 2024-11-26T23:34:11,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/a8fd74c10886444db53616a44fc39259 2024-11-26T23:34:11,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/e92ee4769a6e4f328daf023fc1ec3dec is 50, key is test_row_0/B:col10/1732664050938/Put/seqid=0 2024-11-26T23:34:11,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742054_1230 (size=12001) 2024-11-26T23:34:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:11,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664111267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664111270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664111271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664111272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664111278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664111382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664111382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664111382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664111382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664111391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T23:34:11,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664111592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664111592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664111592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664111592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664111601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/e92ee4769a6e4f328daf023fc1ec3dec 2024-11-26T23:34:11,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/468c9eece52945a08a2d335d0750031b is 50, key is test_row_0/C:col10/1732664050938/Put/seqid=0 2024-11-26T23:34:11,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742055_1231 (size=12001) 2024-11-26T23:34:11,666 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/468c9eece52945a08a2d335d0750031b 2024-11-26T23:34:11,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/a8fd74c10886444db53616a44fc39259 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/a8fd74c10886444db53616a44fc39259 2024-11-26T23:34:11,675 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/a8fd74c10886444db53616a44fc39259, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T23:34:11,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/e92ee4769a6e4f328daf023fc1ec3dec as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e92ee4769a6e4f328daf023fc1ec3dec 2024-11-26T23:34:11,680 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e92ee4769a6e4f328daf023fc1ec3dec, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T23:34:11,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/468c9eece52945a08a2d335d0750031b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/468c9eece52945a08a2d335d0750031b 2024-11-26T23:34:11,685 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/468c9eece52945a08a2d335d0750031b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T23:34:11,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for c5301f81964215ffd1194b5e2f86d347 in 484ms, sequenceid=37, compaction requested=false 2024-11-26T23:34:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-26T23:34:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-26T23:34:11,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T23:34:11,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 793 msec 2024-11-26T23:34:11,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 803 msec 2024-11-26T23:34:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:11,904 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:11,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-26T23:34:11,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/6c066bb7f2744c6b9b2b19526038b5c5 is 50, key is test_row_0/A:col10/1732664051269/Put/seqid=0 2024-11-26T23:34:11,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742056_1232 (size=14341) 2024-11-26T23:34:11,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664111926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664111927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664111927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664111929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664111930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T23:34:12,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:12,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-26T23:34:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T23:34:12,034 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:12,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:12,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664112041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664112041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664112042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664112042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664112042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T23:34:12,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:12,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664112251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664112251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664112252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664112252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664112252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/6c066bb7f2744c6b9b2b19526038b5c5 2024-11-26T23:34:12,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/35bf09f1adb6468a8c530535bc58211b is 50, key is test_row_0/B:col10/1732664051269/Put/seqid=0 2024-11-26T23:34:12,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742057_1233 (size=12001) 2024-11-26T23:34:12,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T23:34:12,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664112561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664112561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664112561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664112561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664112561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:12,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T23:34:12,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/35bf09f1adb6468a8c530535bc58211b 2024-11-26T23:34:12,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/374d539db06c4ea98c03e3f0d35cef5b is 50, key is test_row_0/C:col10/1732664051269/Put/seqid=0 2024-11-26T23:34:12,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742058_1234 (size=12001) 2024-11-26T23:34:12,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:12,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664113071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:13,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664113071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:13,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664113072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664113072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:13,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664113072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:13,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:13,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:13,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:13,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/374d539db06c4ea98c03e3f0d35cef5b 2024-11-26T23:34:13,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/6c066bb7f2744c6b9b2b19526038b5c5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/6c066bb7f2744c6b9b2b19526038b5c5 2024-11-26T23:34:13,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/6c066bb7f2744c6b9b2b19526038b5c5, entries=200, sequenceid=52, filesize=14.0 K 2024-11-26T23:34:13,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/35bf09f1adb6468a8c530535bc58211b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/35bf09f1adb6468a8c530535bc58211b 2024-11-26T23:34:13,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/35bf09f1adb6468a8c530535bc58211b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-26T23:34:13,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/374d539db06c4ea98c03e3f0d35cef5b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/374d539db06c4ea98c03e3f0d35cef5b 2024-11-26T23:34:13,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/374d539db06c4ea98c03e3f0d35cef5b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-26T23:34:13,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c5301f81964215ffd1194b5e2f86d347 in 1257ms, sequenceid=52, compaction requested=true 2024-11-26T23:34:13,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:13,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:13,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:13,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:13,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:13,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:13,161 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:13,161 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:13,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:13,162 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:13,162 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:13,163 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:13,163 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:13,163 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:13,163 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:13,163 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/0496a22283064e229ab1e49a199bcdaf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e92ee4769a6e4f328daf023fc1ec3dec, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/35bf09f1adb6468a8c530535bc58211b] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=35.2 K 2024-11-26T23:34:13,163 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/4ddb75a354124179a30fae6c19a41bdc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/a8fd74c10886444db53616a44fc39259, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/6c066bb7f2744c6b9b2b19526038b5c5] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=39.7 K 2024-11-26T23:34:13,163 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0496a22283064e229ab1e49a199bcdaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732664050893 2024-11-26T23:34:13,163 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ddb75a354124179a30fae6c19a41bdc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732664050893 2024-11-26T23:34:13,163 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e92ee4769a6e4f328daf023fc1ec3dec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732664050933 2024-11-26T23:34:13,164 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8fd74c10886444db53616a44fc39259, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732664050933 2024-11-26T23:34:13,164 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 35bf09f1adb6468a8c530535bc58211b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664051269 2024-11-26T23:34:13,164 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c066bb7f2744c6b9b2b19526038b5c5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664051269 2024-11-26T23:34:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T23:34:13,186 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:13,187 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/5add2a0f3f244ef1aa50931607516b11 is 50, key is test_row_0/A:col10/1732664051269/Put/seqid=0 2024-11-26T23:34:13,189 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:13,190 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b97ff1aae7364a4ba9bc3c10bad4b594 is 50, key is test_row_0/B:col10/1732664051269/Put/seqid=0 2024-11-26T23:34:13,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742059_1235 (size=12104) 2024-11-26T23:34:13,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742060_1236 (size=12104) 2024-11-26T23:34:13,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T23:34:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:13,260 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-26T23:34:13,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/cc2656c5371c4e5495eb18d636626fd1 is 50, key is test_row_0/A:col10/1732664051926/Put/seqid=0 2024-11-26T23:34:13,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742061_1237 (size=12001) 2024-11-26T23:34:13,270 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/cc2656c5371c4e5495eb18d636626fd1 2024-11-26T23:34:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/4c078b74ae44474fb3713d518b80fecc is 50, key is test_row_0/B:col10/1732664051926/Put/seqid=0 2024-11-26T23:34:13,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742062_1238 (size=12001) 2024-11-26T23:34:13,295 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/4c078b74ae44474fb3713d518b80fecc 2024-11-26T23:34:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/ef91ffc4329d4f96812633d252a19eb9 is 50, key is test_row_0/C:col10/1732664051926/Put/seqid=0 2024-11-26T23:34:13,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742063_1239 (size=12001) 2024-11-26T23:34:13,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/ef91ffc4329d4f96812633d252a19eb9 2024-11-26T23:34:13,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/cc2656c5371c4e5495eb18d636626fd1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cc2656c5371c4e5495eb18d636626fd1 2024-11-26T23:34:13,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cc2656c5371c4e5495eb18d636626fd1, entries=150, sequenceid=73, filesize=11.7 K 2024-11-26T23:34:13,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/4c078b74ae44474fb3713d518b80fecc as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4c078b74ae44474fb3713d518b80fecc 2024-11-26T23:34:13,374 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4c078b74ae44474fb3713d518b80fecc, entries=150, sequenceid=73, filesize=11.7 K 2024-11-26T23:34:13,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/ef91ffc4329d4f96812633d252a19eb9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/ef91ffc4329d4f96812633d252a19eb9 2024-11-26T23:34:13,384 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/ef91ffc4329d4f96812633d252a19eb9, entries=150, sequenceid=73, filesize=11.7 K 2024-11-26T23:34:13,386 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for c5301f81964215ffd1194b5e2f86d347 in 126ms, sequenceid=73, compaction requested=true 2024-11-26T23:34:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-26T23:34:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-26T23:34:13,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T23:34:13,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3530 sec 2024-11-26T23:34:13,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3590 sec 2024-11-26T23:34:13,601 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/5add2a0f3f244ef1aa50931607516b11 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/5add2a0f3f244ef1aa50931607516b11 2024-11-26T23:34:13,608 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into 5add2a0f3f244ef1aa50931607516b11(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:13,608 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:13,609 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664053161; duration=0sec 2024-11-26T23:34:13,609 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:13,609 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:13,609 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:13,613 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:13,613 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:13,613 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:13,613 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/cf386b5fa760412aaffa14dd4eed42a7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/468c9eece52945a08a2d335d0750031b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/374d539db06c4ea98c03e3f0d35cef5b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/ef91ffc4329d4f96812633d252a19eb9] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=46.9 K 2024-11-26T23:34:13,614 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf386b5fa760412aaffa14dd4eed42a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732664050893 2024-11-26T23:34:13,614 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 468c9eece52945a08a2d335d0750031b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732664050933 2024-11-26T23:34:13,616 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 374d539db06c4ea98c03e3f0d35cef5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664051269 2024-11-26T23:34:13,616 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b97ff1aae7364a4ba9bc3c10bad4b594 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b97ff1aae7364a4ba9bc3c10bad4b594 2024-11-26T23:34:13,618 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef91ffc4329d4f96812633d252a19eb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732664051926 2024-11-26T23:34:13,633 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into b97ff1aae7364a4ba9bc3c10bad4b594(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:13,633 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:13,633 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664053161; duration=0sec 2024-11-26T23:34:13,633 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:13,633 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:13,655 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:13,655 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9bba5415f8894d6fa99c647bae0eb5bd is 50, key is test_row_0/C:col10/1732664051926/Put/seqid=0 2024-11-26T23:34:13,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742064_1240 (size=12139) 2024-11-26T23:34:13,709 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9bba5415f8894d6fa99c647bae0eb5bd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9bba5415f8894d6fa99c647bae0eb5bd 2024-11-26T23:34:13,716 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into 9bba5415f8894d6fa99c647bae0eb5bd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:13,716 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:13,716 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=12, startTime=1732664053161; duration=0sec 2024-11-26T23:34:13,717 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:13,717 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:13,826 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:34:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:14,089 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:14,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:14,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2e5b10410a6a43cab23102a341d76bc4 is 50, key is test_row_0/A:col10/1732664054088/Put/seqid=0 2024-11-26T23:34:14,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742065_1241 (size=12001) 2024-11-26T23:34:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664114164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664114164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664114165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664114164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664114168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T23:34:14,182 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:14,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-26T23:34:14,186 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:14,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:14,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T23:34:14,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664114282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664114282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664114282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664114282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664114282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T23:34:14,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T23:34:14,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T23:34:14,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:14,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664114491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664114491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664114491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664114491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664114492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T23:34:14,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2e5b10410a6a43cab23102a341d76bc4 2024-11-26T23:34:14,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c6a59340183942dfb31083d1cf34f86a is 50, key is test_row_0/B:col10/1732664054088/Put/seqid=0 2024-11-26T23:34:14,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742066_1242 (size=12001) 2024-11-26T23:34:14,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T23:34:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T23:34:14,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:14,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664114801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664114801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664114801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664114801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664114801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T23:34:14,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c6a59340183942dfb31083d1cf34f86a 2024-11-26T23:34:14,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9ec69f0371fd49feadb27d5bd1a2535a is 50, key is test_row_0/C:col10/1732664054088/Put/seqid=0 2024-11-26T23:34:14,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742067_1243 (size=12001) 2024-11-26T23:34:14,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9ec69f0371fd49feadb27d5bd1a2535a 2024-11-26T23:34:14,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2e5b10410a6a43cab23102a341d76bc4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2e5b10410a6a43cab23102a341d76bc4 2024-11-26T23:34:14,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2e5b10410a6a43cab23102a341d76bc4, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T23:34:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c6a59340183942dfb31083d1cf34f86a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c6a59340183942dfb31083d1cf34f86a 2024-11-26T23:34:14,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T23:34:14,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:14,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c6a59340183942dfb31083d1cf34f86a, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T23:34:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:14,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9ec69f0371fd49feadb27d5bd1a2535a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec69f0371fd49feadb27d5bd1a2535a 2024-11-26T23:34:14,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec69f0371fd49feadb27d5bd1a2535a, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T23:34:14,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c5301f81964215ffd1194b5e2f86d347 in 871ms, sequenceid=87, compaction requested=true 2024-11-26T23:34:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:14,961 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:14,961 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:14,962 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:14,962 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:14,962 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,962 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/5add2a0f3f244ef1aa50931607516b11, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cc2656c5371c4e5495eb18d636626fd1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2e5b10410a6a43cab23102a341d76bc4] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=35.3 K 2024-11-26T23:34:14,962 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:14,962 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:14,962 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5add2a0f3f244ef1aa50931607516b11, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664051269 2024-11-26T23:34:14,962 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:14,962 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b97ff1aae7364a4ba9bc3c10bad4b594, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4c078b74ae44474fb3713d518b80fecc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c6a59340183942dfb31083d1cf34f86a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=35.3 K 2024-11-26T23:34:14,963 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc2656c5371c4e5495eb18d636626fd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732664051926 2024-11-26T23:34:14,963 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b97ff1aae7364a4ba9bc3c10bad4b594, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732664051269 2024-11-26T23:34:14,963 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e5b10410a6a43cab23102a341d76bc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732664054083 2024-11-26T23:34:14,963 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c078b74ae44474fb3713d518b80fecc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732664051926 2024-11-26T23:34:14,964 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a59340183942dfb31083d1cf34f86a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732664054083 2024-11-26T23:34:14,972 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:14,972 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/bbe93b8250ce43c1a737235ecbef953a is 50, key is test_row_0/B:col10/1732664054088/Put/seqid=0 2024-11-26T23:34:14,973 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:14,973 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/aad558e122ff4cf6ab8fd6cb9bdde0d3 is 50, key is test_row_0/A:col10/1732664054088/Put/seqid=0 2024-11-26T23:34:14,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742068_1244 (size=12207) 2024-11-26T23:34:15,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742069_1245 (size=12207) 2024-11-26T23:34:15,007 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/bbe93b8250ce43c1a737235ecbef953a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/bbe93b8250ce43c1a737235ecbef953a 2024-11-26T23:34:15,013 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into bbe93b8250ce43c1a737235ecbef953a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:15,013 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:15,013 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664054961; duration=0sec 2024-11-26T23:34:15,013 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:15,013 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:15,014 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T23:34:15,014 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T23:34:15,014 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T23:34:15,017 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. because compaction request was cancelled 2024-11-26T23:34:15,017 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:15,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T23:34:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:15,108 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:34:15,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/9bffba8dbf1b4dfca05dbfc995130698 is 50, key is test_row_0/A:col10/1732664054163/Put/seqid=0 2024-11-26T23:34:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742070_1246 (size=12001) 2024-11-26T23:34:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:15,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T23:34:15,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664115323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664115328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664115329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664115329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664115330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,419 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/aad558e122ff4cf6ab8fd6cb9bdde0d3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/aad558e122ff4cf6ab8fd6cb9bdde0d3 2024-11-26T23:34:15,427 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into aad558e122ff4cf6ab8fd6cb9bdde0d3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:15,427 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:15,427 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664054961; duration=0sec 2024-11-26T23:34:15,428 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:15,428 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664115432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664115441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664115441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664115441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664115452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,551 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/9bffba8dbf1b4dfca05dbfc995130698 2024-11-26T23:34:15,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b7dc2ef041914dbaa061b28ba9622869 is 50, key is test_row_0/B:col10/1732664054163/Put/seqid=0 2024-11-26T23:34:15,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742071_1247 (size=12001) 2024-11-26T23:34:15,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664115641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664115651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664115651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664115651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664115661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664115952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664115961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664115961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b7dc2ef041914dbaa061b28ba9622869 2024-11-26T23:34:15,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664115961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/c4ba11f5cbfd42539cd6d2d7d7092564 is 50, key is test_row_0/C:col10/1732664054163/Put/seqid=0 2024-11-26T23:34:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664115972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:15,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742072_1248 (size=12001) 2024-11-26T23:34:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T23:34:16,376 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/c4ba11f5cbfd42539cd6d2d7d7092564 2024-11-26T23:34:16,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/9bffba8dbf1b4dfca05dbfc995130698 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/9bffba8dbf1b4dfca05dbfc995130698 2024-11-26T23:34:16,386 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/9bffba8dbf1b4dfca05dbfc995130698, entries=150, sequenceid=113, filesize=11.7 K 2024-11-26T23:34:16,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b7dc2ef041914dbaa061b28ba9622869 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b7dc2ef041914dbaa061b28ba9622869 2024-11-26T23:34:16,390 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b7dc2ef041914dbaa061b28ba9622869, entries=150, sequenceid=113, filesize=11.7 K 2024-11-26T23:34:16,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/c4ba11f5cbfd42539cd6d2d7d7092564 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c4ba11f5cbfd42539cd6d2d7d7092564 2024-11-26T23:34:16,395 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c4ba11f5cbfd42539cd6d2d7d7092564, entries=150, sequenceid=113, filesize=11.7 K 2024-11-26T23:34:16,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for c5301f81964215ffd1194b5e2f86d347 in 1288ms, sequenceid=113, compaction requested=true 2024-11-26T23:34:16,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:16,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:16,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-26T23:34:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-26T23:34:16,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-26T23:34:16,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2100 sec 2024-11-26T23:34:16,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2150 sec 2024-11-26T23:34:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:16,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-26T23:34:16,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0e97e23ef33549bfa2f7362c0467efe2 is 50, key is test_row_0/A:col10/1732664056463/Put/seqid=0 2024-11-26T23:34:16,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742073_1249 (size=14341) 2024-11-26T23:34:16,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664116495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664116495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664116496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664116496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664116500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664116611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664116611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664116612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664116612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664116612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664116822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664116822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664116822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664116822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664116822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:16,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0e97e23ef33549bfa2f7362c0467efe2 2024-11-26T23:34:16,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/4705531fb5d24a2c9e52ef48fbd842b0 is 50, key is test_row_0/B:col10/1732664056463/Put/seqid=0 2024-11-26T23:34:16,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742074_1250 (size=12001) 2024-11-26T23:34:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664117131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664117131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664117131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664117131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664117132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/4705531fb5d24a2c9e52ef48fbd842b0 2024-11-26T23:34:17,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/083e7a2f034c4b82a81b2d6aea508c0f is 50, key is test_row_0/C:col10/1732664056463/Put/seqid=0 2024-11-26T23:34:17,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742075_1251 (size=12001) 2024-11-26T23:34:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664117641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664117641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664117641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664117642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664117642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:17,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/083e7a2f034c4b82a81b2d6aea508c0f 2024-11-26T23:34:17,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0e97e23ef33549bfa2f7362c0467efe2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0e97e23ef33549bfa2f7362c0467efe2 2024-11-26T23:34:17,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0e97e23ef33549bfa2f7362c0467efe2, entries=200, sequenceid=128, filesize=14.0 K 2024-11-26T23:34:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/4705531fb5d24a2c9e52ef48fbd842b0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4705531fb5d24a2c9e52ef48fbd842b0 2024-11-26T23:34:17,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4705531fb5d24a2c9e52ef48fbd842b0, entries=150, sequenceid=128, filesize=11.7 K 2024-11-26T23:34:17,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/083e7a2f034c4b82a81b2d6aea508c0f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/083e7a2f034c4b82a81b2d6aea508c0f 2024-11-26T23:34:17,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/083e7a2f034c4b82a81b2d6aea508c0f, entries=150, sequenceid=128, filesize=11.7 K 2024-11-26T23:34:17,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c5301f81964215ffd1194b5e2f86d347 in 1274ms, sequenceid=128, compaction requested=true 2024-11-26T23:34:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:17,739 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:17,739 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:17,740 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:17,740 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:17,740 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:17,740 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:17,740 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:17,740 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:17,740 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/aad558e122ff4cf6ab8fd6cb9bdde0d3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/9bffba8dbf1b4dfca05dbfc995130698, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0e97e23ef33549bfa2f7362c0467efe2] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=37.6 K 2024-11-26T23:34:17,740 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/bbe93b8250ce43c1a737235ecbef953a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b7dc2ef041914dbaa061b28ba9622869, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4705531fb5d24a2c9e52ef48fbd842b0] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=35.4 K 2024-11-26T23:34:17,741 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad558e122ff4cf6ab8fd6cb9bdde0d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732664054083 2024-11-26T23:34:17,741 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe93b8250ce43c1a737235ecbef953a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732664054083 2024-11-26T23:34:17,742 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b7dc2ef041914dbaa061b28ba9622869, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732664054125 2024-11-26T23:34:17,742 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bffba8dbf1b4dfca05dbfc995130698, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732664054125 2024-11-26T23:34:17,742 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e97e23ef33549bfa2f7362c0467efe2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732664055327 2024-11-26T23:34:17,742 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4705531fb5d24a2c9e52ef48fbd842b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732664055327 2024-11-26T23:34:17,752 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:17,752 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:17,752 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/8447232c649f40ef8a8245cd594c5ec2 is 50, key is test_row_0/B:col10/1732664056463/Put/seqid=0 2024-11-26T23:34:17,752 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0af4a2f5592a4be09997823557e0414c is 50, key is test_row_0/A:col10/1732664056463/Put/seqid=0 2024-11-26T23:34:17,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742076_1252 (size=12309) 2024-11-26T23:34:17,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742077_1253 (size=12309) 2024-11-26T23:34:17,773 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/8447232c649f40ef8a8245cd594c5ec2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8447232c649f40ef8a8245cd594c5ec2 2024-11-26T23:34:17,779 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into 8447232c649f40ef8a8245cd594c5ec2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:17,779 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:17,779 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664057739; duration=0sec 2024-11-26T23:34:17,779 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:17,779 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:17,779 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:17,780 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:17,780 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:17,782 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:17,782 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9bba5415f8894d6fa99c647bae0eb5bd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec69f0371fd49feadb27d5bd1a2535a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c4ba11f5cbfd42539cd6d2d7d7092564, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/083e7a2f034c4b82a81b2d6aea508c0f] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=47.0 K 2024-11-26T23:34:17,782 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bba5415f8894d6fa99c647bae0eb5bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732664051926 2024-11-26T23:34:17,783 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec69f0371fd49feadb27d5bd1a2535a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732664054083 2024-11-26T23:34:17,783 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c4ba11f5cbfd42539cd6d2d7d7092564, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732664054125 2024-11-26T23:34:17,783 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 083e7a2f034c4b82a81b2d6aea508c0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732664055327 2024-11-26T23:34:17,792 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:17,792 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/a2222e5beea44f1cbd39dd6802fc8461 is 50, key is test_row_0/C:col10/1732664056463/Put/seqid=0 2024-11-26T23:34:17,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742078_1254 (size=12275) 2024-11-26T23:34:17,803 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/a2222e5beea44f1cbd39dd6802fc8461 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/a2222e5beea44f1cbd39dd6802fc8461 2024-11-26T23:34:17,808 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into a2222e5beea44f1cbd39dd6802fc8461(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:17,808 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:17,808 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=12, startTime=1732664057739; duration=0sec 2024-11-26T23:34:17,808 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:17,808 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:18,173 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0af4a2f5592a4be09997823557e0414c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0af4a2f5592a4be09997823557e0414c 2024-11-26T23:34:18,178 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into 0af4a2f5592a4be09997823557e0414c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:18,178 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:18,178 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664057739; duration=0sec 2024-11-26T23:34:18,178 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:18,178 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T23:34:18,352 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:18,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-26T23:34:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T23:34:18,355 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:18,355 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:18,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T23:34:18,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T23:34:18,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:18,508 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-26T23:34:18,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/f023d930e42949d1809771e9083a57b5 is 50, key is test_row_0/A:col10/1732664056499/Put/seqid=0 2024-11-26T23:34:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742079_1255 (size=12151) 2024-11-26T23:34:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:18,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:18,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664118666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664118667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T23:34:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664118667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664118667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664118668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664118781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664118781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664118782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664118782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664118782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:18,919 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/f023d930e42949d1809771e9083a57b5 2024-11-26T23:34:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c1c5999dff9c434eb5f2ee3fdeaec83f is 50, key is test_row_0/B:col10/1732664056499/Put/seqid=0 2024-11-26T23:34:18,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742080_1256 (size=12151) 2024-11-26T23:34:18,936 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c1c5999dff9c434eb5f2ee3fdeaec83f 2024-11-26T23:34:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/d0ba524de75c401dad0a8fe480bc56b3 is 50, key is test_row_0/C:col10/1732664056499/Put/seqid=0 2024-11-26T23:34:18,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742081_1257 (size=12151) 2024-11-26T23:34:18,951 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/d0ba524de75c401dad0a8fe480bc56b3 2024-11-26T23:34:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/f023d930e42949d1809771e9083a57b5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f023d930e42949d1809771e9083a57b5 2024-11-26T23:34:18,959 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f023d930e42949d1809771e9083a57b5, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T23:34:18,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c1c5999dff9c434eb5f2ee3fdeaec83f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c1c5999dff9c434eb5f2ee3fdeaec83f 2024-11-26T23:34:18,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c1c5999dff9c434eb5f2ee3fdeaec83f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T23:34:18,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/d0ba524de75c401dad0a8fe480bc56b3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/d0ba524de75c401dad0a8fe480bc56b3 2024-11-26T23:34:18,968 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/d0ba524de75c401dad0a8fe480bc56b3, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T23:34:18,969 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for c5301f81964215ffd1194b5e2f86d347 in 461ms, sequenceid=153, compaction requested=false 2024-11-26T23:34:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-26T23:34:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-26T23:34:18,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T23:34:18,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 616 msec 2024-11-26T23:34:18,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 620 msec 2024-11-26T23:34:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T23:34:18,982 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:18,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-26T23:34:18,984 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T23:34:18,985 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:18,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:18,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-26T23:34:19,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/232acf976bc54b44b31f6060e29b986e is 50, key is test_row_0/A:col10/1732664058994/Put/seqid=0 2024-11-26T23:34:19,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742082_1258 (size=12151) 2024-11-26T23:34:19,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664119013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664119013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664119015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664119017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664119019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T23:34:19,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664119121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664119121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664119132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664119132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664119132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T23:34:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T23:34:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T23:34:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664119331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664119331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664119341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664119342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664119342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/232acf976bc54b44b31f6060e29b986e 2024-11-26T23:34:19,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/1cc03dd4144f4212b2cf5338b27e8acd is 50, key is test_row_0/B:col10/1732664058994/Put/seqid=0 2024-11-26T23:34:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742083_1259 (size=12151) 2024-11-26T23:34:19,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T23:34:19,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:19,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T23:34:19,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:19,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T23:34:19,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664119641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664119641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664119651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664119651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664119652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:19,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T23:34:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:19,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/1cc03dd4144f4212b2cf5338b27e8acd 2024-11-26T23:34:19,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9352264c416943a9949e95466ba21458 is 50, key is test_row_0/C:col10/1732664058994/Put/seqid=0 2024-11-26T23:34:19,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742084_1260 (size=12151) 2024-11-26T23:34:19,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9352264c416943a9949e95466ba21458 2024-11-26T23:34:19,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/232acf976bc54b44b31f6060e29b986e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/232acf976bc54b44b31f6060e29b986e 2024-11-26T23:34:19,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/232acf976bc54b44b31f6060e29b986e, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T23:34:19,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/1cc03dd4144f4212b2cf5338b27e8acd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/1cc03dd4144f4212b2cf5338b27e8acd 2024-11-26T23:34:19,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/1cc03dd4144f4212b2cf5338b27e8acd, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T23:34:19,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9352264c416943a9949e95466ba21458 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9352264c416943a9949e95466ba21458 2024-11-26T23:34:19,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9352264c416943a9949e95466ba21458, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T23:34:19,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for c5301f81964215ffd1194b5e2f86d347 in 860ms, sequenceid=172, compaction requested=true 2024-11-26T23:34:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:19,854 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:19,854 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:19,857 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:19,857 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:19,857 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,858 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0af4a2f5592a4be09997823557e0414c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f023d930e42949d1809771e9083a57b5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/232acf976bc54b44b31f6060e29b986e] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=35.8 K 2024-11-26T23:34:19,858 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:19,858 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:19,858 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,858 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8447232c649f40ef8a8245cd594c5ec2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c1c5999dff9c434eb5f2ee3fdeaec83f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/1cc03dd4144f4212b2cf5338b27e8acd] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=35.8 K 2024-11-26T23:34:19,858 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af4a2f5592a4be09997823557e0414c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732664055327 2024-11-26T23:34:19,859 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8447232c649f40ef8a8245cd594c5ec2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732664055327 2024-11-26T23:34:19,859 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f023d930e42949d1809771e9083a57b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732664056494 2024-11-26T23:34:19,859 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c5999dff9c434eb5f2ee3fdeaec83f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732664056494 2024-11-26T23:34:19,859 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 232acf976bc54b44b31f6060e29b986e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664058665 2024-11-26T23:34:19,860 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc03dd4144f4212b2cf5338b27e8acd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664058665 2024-11-26T23:34:19,869 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:19,870 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/49066b0ee43547b9884325c449304405 is 50, key is test_row_0/A:col10/1732664058994/Put/seqid=0 2024-11-26T23:34:19,871 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:19,871 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/aacad87609c14d5aa21734fef33dc6b4 is 50, key is test_row_0/B:col10/1732664058994/Put/seqid=0 2024-11-26T23:34:19,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742085_1261 (size=12561) 2024-11-26T23:34:19,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742086_1262 (size=12561) 2024-11-26T23:34:19,898 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/49066b0ee43547b9884325c449304405 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/49066b0ee43547b9884325c449304405 2024-11-26T23:34:19,901 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/aacad87609c14d5aa21734fef33dc6b4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/aacad87609c14d5aa21734fef33dc6b4 2024-11-26T23:34:19,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T23:34:19,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-26T23:34:19,905 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into 49066b0ee43547b9884325c449304405(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:19,905 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:19,905 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664059854; duration=0sec 2024-11-26T23:34:19,906 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:19,906 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:19,906 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:19,907 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:19,907 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:19,908 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:19,908 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/a2222e5beea44f1cbd39dd6802fc8461, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/d0ba524de75c401dad0a8fe480bc56b3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9352264c416943a9949e95466ba21458] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=35.7 K 2024-11-26T23:34:19,908 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into aacad87609c14d5aa21734fef33dc6b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:19,908 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:19,908 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664059854; duration=0sec 2024-11-26T23:34:19,908 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:19,908 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:19,908 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2222e5beea44f1cbd39dd6802fc8461, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732664055327 2024-11-26T23:34:19,908 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ba524de75c401dad0a8fe480bc56b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732664056494 2024-11-26T23:34:19,908 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9352264c416943a9949e95466ba21458, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664058665 2024-11-26T23:34:19,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0fb1f97c55cd483d844c9cf40dfe8c80 is 50, key is test_row_0/A:col10/1732664059001/Put/seqid=0 2024-11-26T23:34:19,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742087_1263 (size=12151) 2024-11-26T23:34:19,916 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:19,916 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/78247c03b5a642009d9e8238b213f356 is 50, key is test_row_0/C:col10/1732664058994/Put/seqid=0 2024-11-26T23:34:19,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742088_1264 (size=12527) 2024-11-26T23:34:19,925 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/78247c03b5a642009d9e8238b213f356 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/78247c03b5a642009d9e8238b213f356 2024-11-26T23:34:19,930 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into 78247c03b5a642009d9e8238b213f356(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:19,930 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:19,930 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=13, startTime=1732664059854; duration=0sec 2024-11-26T23:34:19,930 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:19,930 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T23:34:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:20,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664120170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664120172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664120173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664120174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664120174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664120282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664120282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664120282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664120282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664120282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0fb1f97c55cd483d844c9cf40dfe8c80 2024-11-26T23:34:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/375f544c19af4d93956f49adda1694cb is 50, key is test_row_0/B:col10/1732664059001/Put/seqid=0 2024-11-26T23:34:20,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742089_1265 (size=12151) 2024-11-26T23:34:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664120492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664120492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664120492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664120492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664120492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/375f544c19af4d93956f49adda1694cb 2024-11-26T23:34:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/268776cf0232464eafd281e006f52dc7 is 50, key is test_row_0/C:col10/1732664059001/Put/seqid=0 2024-11-26T23:34:20,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742090_1266 (size=12151) 2024-11-26T23:34:20,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664120802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664120802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664120802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664120802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664120802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:20,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:20,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T23:34:21,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/268776cf0232464eafd281e006f52dc7 2024-11-26T23:34:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0fb1f97c55cd483d844c9cf40dfe8c80 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0fb1f97c55cd483d844c9cf40dfe8c80 2024-11-26T23:34:21,147 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0fb1f97c55cd483d844c9cf40dfe8c80, entries=150, sequenceid=189, filesize=11.9 K 2024-11-26T23:34:21,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/375f544c19af4d93956f49adda1694cb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/375f544c19af4d93956f49adda1694cb 2024-11-26T23:34:21,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/375f544c19af4d93956f49adda1694cb, entries=150, sequenceid=189, filesize=11.9 K 2024-11-26T23:34:21,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/268776cf0232464eafd281e006f52dc7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/268776cf0232464eafd281e006f52dc7 2024-11-26T23:34:21,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/268776cf0232464eafd281e006f52dc7, entries=150, sequenceid=189, filesize=11.9 K 2024-11-26T23:34:21,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c5301f81964215ffd1194b5e2f86d347 in 1255ms, sequenceid=189, compaction requested=false 2024-11-26T23:34:21,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:21,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:21,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-26T23:34:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-26T23:34:21,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-26T23:34:21,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-26T23:34:21,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1800 sec 2024-11-26T23:34:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:21,314 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:21,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-26T23:34:21,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/7005fde2595246038adb877673efde74 is 50, key is test_row_0/A:col10/1732664061314/Put/seqid=0 2024-11-26T23:34:21,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742091_1267 (size=12151) 2024-11-26T23:34:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664121328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664121328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664121329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664121329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664121331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664121441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664121441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664121442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664121442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664121442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664121651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664121651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664121652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664121652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664121652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/7005fde2595246038adb877673efde74 2024-11-26T23:34:21,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b45bd2d6457249d881a40d9d2cb46584 is 50, key is test_row_0/B:col10/1732664061314/Put/seqid=0 2024-11-26T23:34:21,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742092_1268 (size=12151) 2024-11-26T23:34:21,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b45bd2d6457249d881a40d9d2cb46584 2024-11-26T23:34:21,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/237d09eadc1642aa900d6611d412e2f1 is 50, key is test_row_0/C:col10/1732664061314/Put/seqid=0 2024-11-26T23:34:21,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742093_1269 (size=12151) 2024-11-26T23:34:21,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664121961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664121961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664121961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664121962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664121962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/237d09eadc1642aa900d6611d412e2f1 2024-11-26T23:34:22,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/7005fde2595246038adb877673efde74 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7005fde2595246038adb877673efde74 2024-11-26T23:34:22,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7005fde2595246038adb877673efde74, entries=150, sequenceid=214, filesize=11.9 K 2024-11-26T23:34:22,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b45bd2d6457249d881a40d9d2cb46584 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b45bd2d6457249d881a40d9d2cb46584 2024-11-26T23:34:22,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b45bd2d6457249d881a40d9d2cb46584, entries=150, sequenceid=214, filesize=11.9 K 2024-11-26T23:34:22,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/237d09eadc1642aa900d6611d412e2f1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/237d09eadc1642aa900d6611d412e2f1 2024-11-26T23:34:22,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/237d09eadc1642aa900d6611d412e2f1, entries=150, sequenceid=214, filesize=11.9 K 2024-11-26T23:34:22,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for c5301f81964215ffd1194b5e2f86d347 in 856ms, sequenceid=214, compaction requested=true 2024-11-26T23:34:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:22,170 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:22,170 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:22,172 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:22,172 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:22,172 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:22,172 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:22,172 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:22,172 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:22,172 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/aacad87609c14d5aa21734fef33dc6b4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/375f544c19af4d93956f49adda1694cb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b45bd2d6457249d881a40d9d2cb46584] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.0 K 2024-11-26T23:34:22,172 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/49066b0ee43547b9884325c449304405, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0fb1f97c55cd483d844c9cf40dfe8c80, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7005fde2595246038adb877673efde74] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.0 K 2024-11-26T23:34:22,173 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting aacad87609c14d5aa21734fef33dc6b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664058665 2024-11-26T23:34:22,173 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49066b0ee43547b9884325c449304405, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664058665 2024-11-26T23:34:22,173 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 375f544c19af4d93956f49adda1694cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732664059001 2024-11-26T23:34:22,173 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb1f97c55cd483d844c9cf40dfe8c80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732664059001 2024-11-26T23:34:22,173 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7005fde2595246038adb877673efde74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732664060170 2024-11-26T23:34:22,173 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b45bd2d6457249d881a40d9d2cb46584, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732664060170 2024-11-26T23:34:22,182 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:22,182 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:22,182 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/7279b69ac2074068beb73acf9fb12a0d is 50, key is test_row_0/B:col10/1732664061314/Put/seqid=0 2024-11-26T23:34:22,182 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/fc25103ec80b4c54b486949535e98647 is 50, key is test_row_0/A:col10/1732664061314/Put/seqid=0 2024-11-26T23:34:22,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742095_1271 (size=12663) 2024-11-26T23:34:22,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742094_1270 (size=12663) 2024-11-26T23:34:22,216 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/7279b69ac2074068beb73acf9fb12a0d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/7279b69ac2074068beb73acf9fb12a0d 2024-11-26T23:34:22,222 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into 7279b69ac2074068beb73acf9fb12a0d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:22,222 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:22,222 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664062170; duration=0sec 2024-11-26T23:34:22,223 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:22,223 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:22,223 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:22,223 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:22,224 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:22,224 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:22,224 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/78247c03b5a642009d9e8238b213f356, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/268776cf0232464eafd281e006f52dc7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/237d09eadc1642aa900d6611d412e2f1] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.0 K 2024-11-26T23:34:22,224 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 78247c03b5a642009d9e8238b213f356, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664058665 2024-11-26T23:34:22,225 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 268776cf0232464eafd281e006f52dc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732664059001 2024-11-26T23:34:22,225 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 237d09eadc1642aa900d6611d412e2f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732664060170 2024-11-26T23:34:22,232 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:22,233 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/f9b74dc9536b4105b2b41a69737b6fd1 is 50, key is test_row_0/C:col10/1732664061314/Put/seqid=0 2024-11-26T23:34:22,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742096_1272 (size=12629) 2024-11-26T23:34:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:22,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:22,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-26T23:34:22,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/713040a469c94b24b0a784d7a61d97b7 is 50, key is test_row_0/A:col10/1732664062474/Put/seqid=0 2024-11-26T23:34:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742097_1273 (size=16931) 2024-11-26T23:34:22,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664122493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664122491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664122494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664122495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664122498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,600 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/fc25103ec80b4c54b486949535e98647 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/fc25103ec80b4c54b486949535e98647 2024-11-26T23:34:22,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664122601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664122601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664122601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664122601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,605 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into fc25103ec80b4c54b486949535e98647(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:22,605 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:22,605 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664062170; duration=0sec 2024-11-26T23:34:22,605 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:22,605 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:22,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664122612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,648 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/f9b74dc9536b4105b2b41a69737b6fd1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f9b74dc9536b4105b2b41a69737b6fd1 2024-11-26T23:34:22,654 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into f9b74dc9536b4105b2b41a69737b6fd1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:22,654 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:22,654 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=13, startTime=1732664062170; duration=0sec 2024-11-26T23:34:22,654 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:22,654 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:22,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664122811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664122811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664122811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664122811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664122822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:22,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/713040a469c94b24b0a784d7a61d97b7 2024-11-26T23:34:22,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/d2dbaec4cf80416e84f94b6588c19a00 is 50, key is test_row_0/B:col10/1732664062474/Put/seqid=0 2024-11-26T23:34:22,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742098_1274 (size=12151) 2024-11-26T23:34:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664123122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664123122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664123122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664123122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664123132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T23:34:23,142 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:23,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-26T23:34:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T23:34:23,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:23,145 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:23,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T23:34:23,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T23:34:23,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:23,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/d2dbaec4cf80416e84f94b6588c19a00 2024-11-26T23:34:23,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/6531e3bb032c465a8d0f57489d7974c5 is 50, key is test_row_0/C:col10/1732664062474/Put/seqid=0 2024-11-26T23:34:23,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742099_1275 (size=12151) 2024-11-26T23:34:23,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T23:34:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T23:34:23,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T23:34:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664123631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664123631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664123631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664123641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664123641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:23,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/6531e3bb032c465a8d0f57489d7974c5 2024-11-26T23:34:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/713040a469c94b24b0a784d7a61d97b7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/713040a469c94b24b0a784d7a61d97b7 2024-11-26T23:34:23,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/713040a469c94b24b0a784d7a61d97b7, entries=250, sequenceid=233, filesize=16.5 K 2024-11-26T23:34:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/d2dbaec4cf80416e84f94b6588c19a00 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d2dbaec4cf80416e84f94b6588c19a00 2024-11-26T23:34:23,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d2dbaec4cf80416e84f94b6588c19a00, entries=150, sequenceid=233, filesize=11.9 K 2024-11-26T23:34:23,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/6531e3bb032c465a8d0f57489d7974c5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6531e3bb032c465a8d0f57489d7974c5 2024-11-26T23:34:23,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T23:34:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:23,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6531e3bb032c465a8d0f57489d7974c5, entries=150, sequenceid=233, filesize=11.9 K 2024-11-26T23:34:23,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c5301f81964215ffd1194b5e2f86d347 in 1287ms, sequenceid=233, compaction requested=false 2024-11-26T23:34:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T23:34:23,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T23:34:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:23,912 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T23:34:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/3191e584c3cd43ac91a626cf8ba3ec33 is 50, key is test_row_0/A:col10/1732664062497/Put/seqid=0 2024-11-26T23:34:23,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742100_1276 (size=12151) 2024-11-26T23:34:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T23:34:24,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/3191e584c3cd43ac91a626cf8ba3ec33 2024-11-26T23:34:24,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b9bf04a4617f442aa625fbd3621b9c7f is 50, key is test_row_0/B:col10/1732664062497/Put/seqid=0 2024-11-26T23:34:24,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742101_1277 (size=12151) 2024-11-26T23:34:24,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b9bf04a4617f442aa625fbd3621b9c7f 2024-11-26T23:34:24,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/4f8ec8918fc947caae9fd92962c320fd is 50, key is test_row_0/C:col10/1732664062497/Put/seqid=0 2024-11-26T23:34:24,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742102_1278 (size=12151) 2024-11-26T23:34:24,352 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/4f8ec8918fc947caae9fd92962c320fd 2024-11-26T23:34:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/3191e584c3cd43ac91a626cf8ba3ec33 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/3191e584c3cd43ac91a626cf8ba3ec33 2024-11-26T23:34:24,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/3191e584c3cd43ac91a626cf8ba3ec33, entries=150, sequenceid=253, filesize=11.9 K 2024-11-26T23:34:24,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b9bf04a4617f442aa625fbd3621b9c7f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b9bf04a4617f442aa625fbd3621b9c7f 2024-11-26T23:34:24,369 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b9bf04a4617f442aa625fbd3621b9c7f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-26T23:34:24,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/4f8ec8918fc947caae9fd92962c320fd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/4f8ec8918fc947caae9fd92962c320fd 2024-11-26T23:34:24,377 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/4f8ec8918fc947caae9fd92962c320fd, entries=150, sequenceid=253, filesize=11.9 K 2024-11-26T23:34:24,378 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for c5301f81964215ffd1194b5e2f86d347 in 466ms, sequenceid=253, compaction requested=true 2024-11-26T23:34:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-26T23:34:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-26T23:34:24,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-26T23:34:24,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2470 sec 2024-11-26T23:34:24,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2530 sec 2024-11-26T23:34:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:24,651 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:24,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:24,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/67b7677ef7cc43008972a0df47e9daab is 50, key is test_row_0/A:col10/1732664064643/Put/seqid=0 2024-11-26T23:34:24,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742103_1279 (size=17181) 2024-11-26T23:34:24,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664124682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664124684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664124686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664124686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664124688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664124791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664124801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664124802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664124802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664124802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664125001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664125011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664125011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664125012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664125012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/67b7677ef7cc43008972a0df47e9daab 2024-11-26T23:34:25,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/d18aefea7354402da951f59a5bfd70a5 is 50, key is test_row_0/B:col10/1732664064643/Put/seqid=0 2024-11-26T23:34:25,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742104_1280 (size=12301) 2024-11-26T23:34:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T23:34:25,292 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:25,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-26T23:34:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T23:34:25,295 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:25,295 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:25,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:25,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664125311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664125321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664125322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664125322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664125322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T23:34:25,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T23:34:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/d18aefea7354402da951f59a5bfd70a5 2024-11-26T23:34:25,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/1c506437ec9247ca8e8976689bdff17c is 50, key is test_row_0/C:col10/1732664064643/Put/seqid=0 2024-11-26T23:34:25,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742105_1281 (size=12301) 2024-11-26T23:34:25,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T23:34:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T23:34:25,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T23:34:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664125822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664125832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664125832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664125832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664125832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:25,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T23:34:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:25,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/1c506437ec9247ca8e8976689bdff17c 2024-11-26T23:34:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/67b7677ef7cc43008972a0df47e9daab as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/67b7677ef7cc43008972a0df47e9daab 2024-11-26T23:34:25,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/67b7677ef7cc43008972a0df47e9daab, entries=250, sequenceid=265, filesize=16.8 K 2024-11-26T23:34:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/d18aefea7354402da951f59a5bfd70a5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d18aefea7354402da951f59a5bfd70a5 2024-11-26T23:34:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T23:34:25,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d18aefea7354402da951f59a5bfd70a5, entries=150, sequenceid=265, filesize=12.0 K 2024-11-26T23:34:25,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/1c506437ec9247ca8e8976689bdff17c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1c506437ec9247ca8e8976689bdff17c 2024-11-26T23:34:25,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1c506437ec9247ca8e8976689bdff17c, entries=150, sequenceid=265, filesize=12.0 K 2024-11-26T23:34:25,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c5301f81964215ffd1194b5e2f86d347 in 1276ms, sequenceid=265, compaction requested=true 2024-11-26T23:34:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:25,927 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:25,927 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:25,929 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:25,929 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:25,929 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58926 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:25,929 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:25,929 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,929 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:25,929 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/7279b69ac2074068beb73acf9fb12a0d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d2dbaec4cf80416e84f94b6588c19a00, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b9bf04a4617f442aa625fbd3621b9c7f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d18aefea7354402da951f59a5bfd70a5] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=48.1 K 2024-11-26T23:34:25,929 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/fc25103ec80b4c54b486949535e98647, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/713040a469c94b24b0a784d7a61d97b7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/3191e584c3cd43ac91a626cf8ba3ec33, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/67b7677ef7cc43008972a0df47e9daab] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=57.5 K 2024-11-26T23:34:25,929 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 7279b69ac2074068beb73acf9fb12a0d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732664060170 2024-11-26T23:34:25,930 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc25103ec80b4c54b486949535e98647, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732664060170 2024-11-26T23:34:25,930 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d2dbaec4cf80416e84f94b6588c19a00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732664062474 2024-11-26T23:34:25,930 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713040a469c94b24b0a784d7a61d97b7, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732664061328 2024-11-26T23:34:25,930 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b9bf04a4617f442aa625fbd3621b9c7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664062491 2024-11-26T23:34:25,930 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3191e584c3cd43ac91a626cf8ba3ec33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664062491 2024-11-26T23:34:25,931 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d18aefea7354402da951f59a5bfd70a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732664064643 2024-11-26T23:34:25,931 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67b7677ef7cc43008972a0df47e9daab, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732664064643 2024-11-26T23:34:25,940 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:25,941 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/5a97b58b687c46fdb85fa93ef49c54cf is 50, key is test_row_0/B:col10/1732664064643/Put/seqid=0 2024-11-26T23:34:25,943 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:25,944 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/7e7aeda104264005bd58716c2f63dd0e is 50, key is test_row_0/A:col10/1732664064643/Put/seqid=0 2024-11-26T23:34:25,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742107_1283 (size=12949) 2024-11-26T23:34:25,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742106_1282 (size=12949) 2024-11-26T23:34:26,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T23:34:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:26,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:34:26,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/10fd5c3c9a3248699d8ccacf6fde50b6 is 50, key is test_row_0/A:col10/1732664064685/Put/seqid=0 2024-11-26T23:34:26,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742108_1284 (size=12301) 2024-11-26T23:34:26,356 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/5a97b58b687c46fdb85fa93ef49c54cf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5a97b58b687c46fdb85fa93ef49c54cf 2024-11-26T23:34:26,356 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/7e7aeda104264005bd58716c2f63dd0e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7e7aeda104264005bd58716c2f63dd0e 2024-11-26T23:34:26,361 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into 5a97b58b687c46fdb85fa93ef49c54cf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:26,361 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:26,361 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=12, startTime=1732664065927; duration=0sec 2024-11-26T23:34:26,361 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into 7e7aeda104264005bd58716c2f63dd0e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:26,361 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:26,361 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=12, startTime=1732664065927; duration=0sec 2024-11-26T23:34:26,361 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:26,361 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:26,361 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:26,361 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:26,361 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:26,363 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:26,363 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:26,363 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:26,363 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f9b74dc9536b4105b2b41a69737b6fd1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6531e3bb032c465a8d0f57489d7974c5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/4f8ec8918fc947caae9fd92962c320fd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1c506437ec9247ca8e8976689bdff17c] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=48.1 K 2024-11-26T23:34:26,363 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b74dc9536b4105b2b41a69737b6fd1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732664060170 2024-11-26T23:34:26,364 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6531e3bb032c465a8d0f57489d7974c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732664062474 2024-11-26T23:34:26,364 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8ec8918fc947caae9fd92962c320fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664062491 2024-11-26T23:34:26,364 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c506437ec9247ca8e8976689bdff17c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732664064643 2024-11-26T23:34:26,372 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:26,373 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/f0480f2bb826488e992a5296f33b6e49 is 50, key is test_row_0/C:col10/1732664064643/Put/seqid=0 2024-11-26T23:34:26,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742109_1285 (size=12915) 2024-11-26T23:34:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T23:34:26,470 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/10fd5c3c9a3248699d8ccacf6fde50b6 2024-11-26T23:34:26,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c2fa3bc1cfc340cebbb6573db8eedce0 is 50, key is test_row_0/B:col10/1732664064685/Put/seqid=0 2024-11-26T23:34:26,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742110_1286 (size=12301) 2024-11-26T23:34:26,782 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/f0480f2bb826488e992a5296f33b6e49 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f0480f2bb826488e992a5296f33b6e49 2024-11-26T23:34:26,787 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into f0480f2bb826488e992a5296f33b6e49(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:26,787 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:26,787 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=12, startTime=1732664065927; duration=0sec 2024-11-26T23:34:26,787 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:26,787 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:26,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:26,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664126847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664126851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664126851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,855 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in c5301f81964215ffd1194b5e2f86d347 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664126852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664126852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c2fa3bc1cfc340cebbb6573db8eedce0 2024-11-26T23:34:26,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/7917682e0de748c08acd744f298472e7 is 50, key is test_row_0/C:col10/1732664064685/Put/seqid=0 2024-11-26T23:34:26,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742111_1287 (size=12301) 2024-11-26T23:34:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664126962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664126962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664126962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664126962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664127171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664127171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664127172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664127172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/7917682e0de748c08acd744f298472e7 2024-11-26T23:34:27,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/10fd5c3c9a3248699d8ccacf6fde50b6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/10fd5c3c9a3248699d8ccacf6fde50b6 2024-11-26T23:34:27,313 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/10fd5c3c9a3248699d8ccacf6fde50b6, entries=150, sequenceid=289, filesize=12.0 K 2024-11-26T23:34:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c2fa3bc1cfc340cebbb6573db8eedce0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2fa3bc1cfc340cebbb6573db8eedce0 2024-11-26T23:34:27,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2fa3bc1cfc340cebbb6573db8eedce0, entries=150, sequenceid=289, filesize=12.0 K 2024-11-26T23:34:27,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/7917682e0de748c08acd744f298472e7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/7917682e0de748c08acd744f298472e7 2024-11-26T23:34:27,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/7917682e0de748c08acd744f298472e7, entries=150, sequenceid=289, filesize=12.0 K 2024-11-26T23:34:27,324 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for c5301f81964215ffd1194b5e2f86d347 in 1264ms, sequenceid=289, compaction requested=false 2024-11-26T23:34:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-26T23:34:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-26T23:34:27,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-26T23:34:27,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0300 sec 2024-11-26T23:34:27,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0350 sec 2024-11-26T23:34:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T23:34:27,442 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:27,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-26T23:34:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T23:34:27,445 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:27,445 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:27,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:27,484 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:27,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-26T23:34:27,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/89b8e50a23ec4b58b147fb67810c6484 is 50, key is test_row_0/A:col10/1732664066851/Put/seqid=0 2024-11-26T23:34:27,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742112_1288 (size=12301) 2024-11-26T23:34:27,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664127507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664127507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664127510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664127512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T23:34:27,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T23:34:27,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:27,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:27,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:27,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664127622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664127622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664127622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664127622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T23:34:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:27,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:27,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:27,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T23:34:27,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664127832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664127832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664127832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664127832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:27,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/89b8e50a23ec4b58b147fb67810c6484 2024-11-26T23:34:27,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/addd667d1efa4aae9777a29857b67a97 is 50, key is test_row_0/B:col10/1732664066851/Put/seqid=0 2024-11-26T23:34:27,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T23:34:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:27,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:27,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742113_1289 (size=12301) 2024-11-26T23:34:28,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T23:34:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T23:34:28,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664128142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664128142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664128142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664128142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T23:34:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/addd667d1efa4aae9777a29857b67a97 2024-11-26T23:34:28,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/64c96feeebf146a2b28f08b49fbb9f52 is 50, key is test_row_0/C:col10/1732664066851/Put/seqid=0 2024-11-26T23:34:28,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742114_1290 (size=12301) 2024-11-26T23:34:28,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/64c96feeebf146a2b28f08b49fbb9f52 2024-11-26T23:34:28,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/89b8e50a23ec4b58b147fb67810c6484 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/89b8e50a23ec4b58b147fb67810c6484 2024-11-26T23:34:28,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/89b8e50a23ec4b58b147fb67810c6484, entries=150, sequenceid=307, filesize=12.0 K 2024-11-26T23:34:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/addd667d1efa4aae9777a29857b67a97 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/addd667d1efa4aae9777a29857b67a97 2024-11-26T23:34:28,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/addd667d1efa4aae9777a29857b67a97, entries=150, sequenceid=307, filesize=12.0 K 2024-11-26T23:34:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/64c96feeebf146a2b28f08b49fbb9f52 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/64c96feeebf146a2b28f08b49fbb9f52 2024-11-26T23:34:28,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/64c96feeebf146a2b28f08b49fbb9f52, entries=150, sequenceid=307, filesize=12.0 K 2024-11-26T23:34:28,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c5301f81964215ffd1194b5e2f86d347 in 857ms, sequenceid=307, compaction requested=true 2024-11-26T23:34:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:28,342 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:28,342 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:28,345 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:28,345 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:28,345 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:28,345 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:28,345 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,345 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,345 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5a97b58b687c46fdb85fa93ef49c54cf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2fa3bc1cfc340cebbb6573db8eedce0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/addd667d1efa4aae9777a29857b67a97] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.7 K 2024-11-26T23:34:28,345 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7e7aeda104264005bd58716c2f63dd0e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/10fd5c3c9a3248699d8ccacf6fde50b6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/89b8e50a23ec4b58b147fb67810c6484] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.7 K 2024-11-26T23:34:28,345 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a97b58b687c46fdb85fa93ef49c54cf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732664064643 2024-11-26T23:34:28,345 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e7aeda104264005bd58716c2f63dd0e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732664064643 2024-11-26T23:34:28,346 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c2fa3bc1cfc340cebbb6573db8eedce0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732664064682 2024-11-26T23:34:28,346 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10fd5c3c9a3248699d8ccacf6fde50b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732664064682 2024-11-26T23:34:28,346 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting addd667d1efa4aae9777a29857b67a97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732664066851 2024-11-26T23:34:28,347 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b8e50a23ec4b58b147fb67810c6484, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732664066851 2024-11-26T23:34:28,355 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:28,355 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:28,356 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/8454315e8555403997813028e7c45e72 is 50, key is test_row_0/B:col10/1732664066851/Put/seqid=0 2024-11-26T23:34:28,356 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2c91675c429c4f638e58d7e60f7c0eba is 50, key is test_row_0/A:col10/1732664066851/Put/seqid=0 2024-11-26T23:34:28,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T23:34:28,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-26T23:34:28,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742115_1291 (size=13051) 2024-11-26T23:34:28,372 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/8454315e8555403997813028e7c45e72 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8454315e8555403997813028e7c45e72 2024-11-26T23:34:28,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742116_1292 (size=13051) 2024-11-26T23:34:28,377 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into 8454315e8555403997813028e7c45e72(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:28,377 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:28,377 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664068342; duration=0sec 2024-11-26T23:34:28,377 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:28,377 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:28,378 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:28,379 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:28,379 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:28,379 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,380 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f0480f2bb826488e992a5296f33b6e49, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/7917682e0de748c08acd744f298472e7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/64c96feeebf146a2b28f08b49fbb9f52] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.6 K 2024-11-26T23:34:28,380 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f0480f2bb826488e992a5296f33b6e49, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732664064643 2024-11-26T23:34:28,380 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 7917682e0de748c08acd744f298472e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732664064682 2024-11-26T23:34:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2aad309d028d4746ae95d48d03ca58a9 is 50, key is test_row_0/A:col10/1732664067503/Put/seqid=0 2024-11-26T23:34:28,381 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 64c96feeebf146a2b28f08b49fbb9f52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732664066851 2024-11-26T23:34:28,383 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2c91675c429c4f638e58d7e60f7c0eba as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2c91675c429c4f638e58d7e60f7c0eba 2024-11-26T23:34:28,389 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into 2c91675c429c4f638e58d7e60f7c0eba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:28,389 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:28,389 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664068342; duration=0sec 2024-11-26T23:34:28,389 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:28,389 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:28,395 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:28,395 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/c11bd46ab8214285a9714901d45ce07f is 50, key is test_row_0/C:col10/1732664066851/Put/seqid=0 2024-11-26T23:34:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742117_1293 (size=12301) 2024-11-26T23:34:28,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2aad309d028d4746ae95d48d03ca58a9 2024-11-26T23:34:28,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742118_1294 (size=13017) 2024-11-26T23:34:28,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/6e37634ad1244391bf2cd91b61e2b8be is 50, key is test_row_0/B:col10/1732664067503/Put/seqid=0 2024-11-26T23:34:28,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742119_1295 (size=12301) 2024-11-26T23:34:28,416 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/6e37634ad1244391bf2cd91b61e2b8be 2024-11-26T23:34:28,420 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/c11bd46ab8214285a9714901d45ce07f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c11bd46ab8214285a9714901d45ce07f 2024-11-26T23:34:28,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/326869def9214a94aab97e4c2b87bb89 is 50, key is test_row_0/C:col10/1732664067503/Put/seqid=0 2024-11-26T23:34:28,429 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into c11bd46ab8214285a9714901d45ce07f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:28,429 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:28,429 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=13, startTime=1732664068342; duration=0sec 2024-11-26T23:34:28,429 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:28,429 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:28,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742120_1296 (size=12301) 2024-11-26T23:34:28,435 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/326869def9214a94aab97e4c2b87bb89 2024-11-26T23:34:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/2aad309d028d4746ae95d48d03ca58a9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2aad309d028d4746ae95d48d03ca58a9 2024-11-26T23:34:28,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2aad309d028d4746ae95d48d03ca58a9, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T23:34:28,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/6e37634ad1244391bf2cd91b61e2b8be as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/6e37634ad1244391bf2cd91b61e2b8be 2024-11-26T23:34:28,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/6e37634ad1244391bf2cd91b61e2b8be, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T23:34:28,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/326869def9214a94aab97e4c2b87bb89 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/326869def9214a94aab97e4c2b87bb89 2024-11-26T23:34:28,458 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/326869def9214a94aab97e4c2b87bb89, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T23:34:28,458 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for c5301f81964215ffd1194b5e2f86d347 in 94ms, sequenceid=328, compaction requested=false 2024-11-26T23:34:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-26T23:34:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-26T23:34:28,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-26T23:34:28,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-26T23:34:28,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0200 sec 2024-11-26T23:34:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T23:34:28,581 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:28,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-26T23:34:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T23:34:28,584 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:28,588 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:28,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:28,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:28,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:28,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/ad7d5c908af0400b8a5f9fd57adf4e58 is 50, key is test_row_0/A:col10/1732664068661/Put/seqid=0 2024-11-26T23:34:28,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742121_1297 (size=14741) 2024-11-26T23:34:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T23:34:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664128687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664128690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664128691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664128693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:28,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:28,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664128802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664128802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664128802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664128802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51284 deadline: 1732664128881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:28,885 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in c5301f81964215ffd1194b5e2f86d347 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:28,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:28,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T23:34:29,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664129012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664129012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664129012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664129012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:29,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:29,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/ad7d5c908af0400b8a5f9fd57adf4e58 2024-11-26T23:34:29,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/478268e4bc14471793bfbd327b51d8d2 is 50, key is test_row_0/B:col10/1732664068661/Put/seqid=0 2024-11-26T23:34:29,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742122_1298 (size=12301) 2024-11-26T23:34:29,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:29,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T23:34:29,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664129321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664129321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664129322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664129322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:29,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:29,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/478268e4bc14471793bfbd327b51d8d2 2024-11-26T23:34:29,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/2932c609067741ce9f40a59073ea2ade is 50, key is test_row_0/C:col10/1732664068661/Put/seqid=0 2024-11-26T23:34:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742123_1299 (size=12301) 2024-11-26T23:34:29,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:29,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:29,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:29,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:29,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T23:34:29,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:29,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. as already flushing 2024-11-26T23:34:29,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:29,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664129831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664129831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664129831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664129831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:29,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/2932c609067741ce9f40a59073ea2ade 2024-11-26T23:34:29,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/ad7d5c908af0400b8a5f9fd57adf4e58 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ad7d5c908af0400b8a5f9fd57adf4e58 2024-11-26T23:34:29,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ad7d5c908af0400b8a5f9fd57adf4e58, entries=200, sequenceid=343, filesize=14.4 K 2024-11-26T23:34:29,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/478268e4bc14471793bfbd327b51d8d2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/478268e4bc14471793bfbd327b51d8d2 2024-11-26T23:34:29,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/478268e4bc14471793bfbd327b51d8d2, entries=150, sequenceid=343, filesize=12.0 K 2024-11-26T23:34:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/2932c609067741ce9f40a59073ea2ade as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/2932c609067741ce9f40a59073ea2ade 2024-11-26T23:34:29,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/2932c609067741ce9f40a59073ea2ade, entries=150, sequenceid=343, filesize=12.0 K 2024-11-26T23:34:29,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c5301f81964215ffd1194b5e2f86d347 in 1252ms, sequenceid=343, compaction requested=true 2024-11-26T23:34:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:29,913 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:29,913 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:29,914 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:29,914 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:29,914 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:29,914 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:29,914 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,914 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,914 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2c91675c429c4f638e58d7e60f7c0eba, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2aad309d028d4746ae95d48d03ca58a9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ad7d5c908af0400b8a5f9fd57adf4e58] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=39.2 K 2024-11-26T23:34:29,914 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8454315e8555403997813028e7c45e72, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/6e37634ad1244391bf2cd91b61e2b8be, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/478268e4bc14471793bfbd327b51d8d2] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.8 K 2024-11-26T23:34:29,915 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c91675c429c4f638e58d7e60f7c0eba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732664066851 2024-11-26T23:34:29,915 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8454315e8555403997813028e7c45e72, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732664066851 2024-11-26T23:34:29,915 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aad309d028d4746ae95d48d03ca58a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664067499 2024-11-26T23:34:29,915 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e37634ad1244391bf2cd91b61e2b8be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664067499 2024-11-26T23:34:29,915 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 478268e4bc14471793bfbd327b51d8d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732664068656 2024-11-26T23:34:29,915 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7d5c908af0400b8a5f9fd57adf4e58, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732664068656 2024-11-26T23:34:29,922 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:29,923 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/ee880bf6577d483b8bbcaa5242c97614 is 50, key is test_row_0/A:col10/1732664068661/Put/seqid=0 2024-11-26T23:34:29,925 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:29,926 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/5d78afacd32c47fdae73dff10d5289ee is 50, key is test_row_0/B:col10/1732664068661/Put/seqid=0 2024-11-26T23:34:29,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742124_1300 (size=13153) 2024-11-26T23:34:29,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742125_1301 (size=13153) 2024-11-26T23:34:29,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T23:34:29,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:29,966 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:34:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/258b1e49d62541ddb064aa8dced40baf is 50, key is test_row_0/A:col10/1732664068684/Put/seqid=0 2024-11-26T23:34:29,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742126_1302 (size=12301) 2024-11-26T23:34:30,337 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/ee880bf6577d483b8bbcaa5242c97614 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ee880bf6577d483b8bbcaa5242c97614 2024-11-26T23:34:30,342 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into ee880bf6577d483b8bbcaa5242c97614(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:30,342 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:30,342 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664069913; duration=0sec 2024-11-26T23:34:30,342 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:30,342 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:30,342 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:30,343 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:30,343 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:30,343 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:30,343 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c11bd46ab8214285a9714901d45ce07f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/326869def9214a94aab97e4c2b87bb89, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/2932c609067741ce9f40a59073ea2ade] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.7 K 2024-11-26T23:34:30,343 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting c11bd46ab8214285a9714901d45ce07f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732664066851 2024-11-26T23:34:30,344 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 326869def9214a94aab97e4c2b87bb89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664067499 2024-11-26T23:34:30,344 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2932c609067741ce9f40a59073ea2ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732664068656 2024-11-26T23:34:30,351 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/5d78afacd32c47fdae73dff10d5289ee as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5d78afacd32c47fdae73dff10d5289ee 2024-11-26T23:34:30,351 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:30,352 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/1e36888305714c749d7692e497ac7cc7 is 50, key is test_row_0/C:col10/1732664068661/Put/seqid=0 2024-11-26T23:34:30,356 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into 5d78afacd32c47fdae73dff10d5289ee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:30,356 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:30,356 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664069913; duration=0sec 2024-11-26T23:34:30,356 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:30,356 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:30,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742127_1303 (size=13119) 2024-11-26T23:34:30,375 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/258b1e49d62541ddb064aa8dced40baf 2024-11-26T23:34:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/8daaf07141db41b9aeaaf5f88c675981 is 50, key is test_row_0/B:col10/1732664068684/Put/seqid=0 2024-11-26T23:34:30,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742128_1304 (size=12301) 2024-11-26T23:34:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T23:34:30,762 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/1e36888305714c749d7692e497ac7cc7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1e36888305714c749d7692e497ac7cc7 2024-11-26T23:34:30,767 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into 1e36888305714c749d7692e497ac7cc7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:30,767 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:30,767 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=13, startTime=1732664069913; duration=0sec 2024-11-26T23:34:30,767 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:30,767 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:30,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/8daaf07141db41b9aeaaf5f88c675981 2024-11-26T23:34:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/b2ff539073044ddf8db1b1d5fe905c63 is 50, key is test_row_0/C:col10/1732664068684/Put/seqid=0 2024-11-26T23:34:30,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742129_1305 (size=12301) 2024-11-26T23:34:30,810 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/b2ff539073044ddf8db1b1d5fe905c63 2024-11-26T23:34:30,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/258b1e49d62541ddb064aa8dced40baf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/258b1e49d62541ddb064aa8dced40baf 2024-11-26T23:34:30,819 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/258b1e49d62541ddb064aa8dced40baf, entries=150, sequenceid=367, filesize=12.0 K 2024-11-26T23:34:30,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/8daaf07141db41b9aeaaf5f88c675981 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8daaf07141db41b9aeaaf5f88c675981 2024-11-26T23:34:30,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8daaf07141db41b9aeaaf5f88c675981, entries=150, sequenceid=367, filesize=12.0 K 2024-11-26T23:34:30,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/b2ff539073044ddf8db1b1d5fe905c63 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/b2ff539073044ddf8db1b1d5fe905c63 2024-11-26T23:34:30,829 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/b2ff539073044ddf8db1b1d5fe905c63, entries=150, sequenceid=367, filesize=12.0 K 2024-11-26T23:34:30,830 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for c5301f81964215ffd1194b5e2f86d347 in 865ms, sequenceid=367, compaction requested=false 2024-11-26T23:34:30,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:30,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:30,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-26T23:34:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-26T23:34:30,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-26T23:34:30,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2450 sec 2024-11-26T23:34:30,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2530 sec 2024-11-26T23:34:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:30,852 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:30,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:30,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/f7d1da867adb4527b27274b127346434 is 50, key is test_row_0/A:col10/1732664070845/Put/seqid=0 2024-11-26T23:34:30,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742130_1306 (size=12301) 2024-11-26T23:34:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664130882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:30,886 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-26T23:34:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,887 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664130883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:30,887 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,887 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:30,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664130886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:30,890 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-26T23:34:30,890 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:30,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,890 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664130886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:30,890 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,890 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-26T23:34:30,890 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:30,890 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,890 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,890 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:30,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:30,891 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-26T23:34:30,891 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:30,891 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,891 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:30,893 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-26T23:34:30,893 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:30,893 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,893 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:30,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664130991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664130991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664131001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664131002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664131201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664131201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664131211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664131211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/f7d1da867adb4527b27274b127346434 2024-11-26T23:34:31,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/10b66dfba66a4fc4a5adf64ceda79596 is 50, key is test_row_0/B:col10/1732664070845/Put/seqid=0 2024-11-26T23:34:31,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742131_1307 (size=12301) 2024-11-26T23:34:31,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664131512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664131512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664131522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664131522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:31,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/10b66dfba66a4fc4a5adf64ceda79596 2024-11-26T23:34:31,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/e995cfb0696d46ef871a8a7b77aa6657 is 50, key is test_row_0/C:col10/1732664070845/Put/seqid=0 2024-11-26T23:34:31,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742132_1308 (size=12301) 2024-11-26T23:34:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51332 deadline: 1732664132021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51276 deadline: 1732664132021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:32,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:32,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51292 deadline: 1732664132032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1732664132032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:34:32,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/e995cfb0696d46ef871a8a7b77aa6657 2024-11-26T23:34:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/f7d1da867adb4527b27274b127346434 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f7d1da867adb4527b27274b127346434 2024-11-26T23:34:32,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f7d1da867adb4527b27274b127346434, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T23:34:32,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/10b66dfba66a4fc4a5adf64ceda79596 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/10b66dfba66a4fc4a5adf64ceda79596 2024-11-26T23:34:32,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/10b66dfba66a4fc4a5adf64ceda79596, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T23:34:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/e995cfb0696d46ef871a8a7b77aa6657 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/e995cfb0696d46ef871a8a7b77aa6657 2024-11-26T23:34:32,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/e995cfb0696d46ef871a8a7b77aa6657, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T23:34:32,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for c5301f81964215ffd1194b5e2f86d347 in 1248ms, sequenceid=381, compaction requested=true 2024-11-26T23:34:32,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:32,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:32,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:32,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:32,100 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:32,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5301f81964215ffd1194b5e2f86d347:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:32,101 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:32,101 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:32,101 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:32,101 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/A is initiating minor compaction (all files) 2024-11-26T23:34:32,101 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/B is initiating minor compaction (all files) 2024-11-26T23:34:32,101 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/A in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:32,101 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/B in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:32,101 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ee880bf6577d483b8bbcaa5242c97614, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/258b1e49d62541ddb064aa8dced40baf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f7d1da867adb4527b27274b127346434] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.9 K 2024-11-26T23:34:32,101 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5d78afacd32c47fdae73dff10d5289ee, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8daaf07141db41b9aeaaf5f88c675981, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/10b66dfba66a4fc4a5adf64ceda79596] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.9 K 2024-11-26T23:34:32,102 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d78afacd32c47fdae73dff10d5289ee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732664068656 2024-11-26T23:34:32,102 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee880bf6577d483b8bbcaa5242c97614, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732664068656 2024-11-26T23:34:32,102 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8daaf07141db41b9aeaaf5f88c675981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664068684 2024-11-26T23:34:32,102 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 258b1e49d62541ddb064aa8dced40baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664068684 2024-11-26T23:34:32,102 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b66dfba66a4fc4a5adf64ceda79596, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732664070842 2024-11-26T23:34:32,102 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7d1da867adb4527b27274b127346434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732664070842 2024-11-26T23:34:32,108 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#A#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:32,108 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:32,108 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0432443df7f949a3ad9b2ed0484fd9cc is 50, key is test_row_0/A:col10/1732664070845/Put/seqid=0 2024-11-26T23:34:32,108 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b330eed277c740b78784fe44da2acf23 is 50, key is test_row_0/B:col10/1732664070845/Put/seqid=0 2024-11-26T23:34:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742134_1310 (size=13255) 2024-11-26T23:34:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742133_1309 (size=13255) 2024-11-26T23:34:32,116 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/0432443df7f949a3ad9b2ed0484fd9cc as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0432443df7f949a3ad9b2ed0484fd9cc 2024-11-26T23:34:32,120 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/A of c5301f81964215ffd1194b5e2f86d347 into 0432443df7f949a3ad9b2ed0484fd9cc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:32,120 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:32,120 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/A, priority=13, startTime=1732664072100; duration=0sec 2024-11-26T23:34:32,120 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:32,120 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:A 2024-11-26T23:34:32,120 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:32,121 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:32,121 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): c5301f81964215ffd1194b5e2f86d347/C is initiating minor compaction (all files) 2024-11-26T23:34:32,121 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5301f81964215ffd1194b5e2f86d347/C in TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:32,121 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1e36888305714c749d7692e497ac7cc7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/b2ff539073044ddf8db1b1d5fe905c63, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/e995cfb0696d46ef871a8a7b77aa6657] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp, totalSize=36.8 K 2024-11-26T23:34:32,121 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e36888305714c749d7692e497ac7cc7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732664068656 2024-11-26T23:34:32,121 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ff539073044ddf8db1b1d5fe905c63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664068684 2024-11-26T23:34:32,122 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting e995cfb0696d46ef871a8a7b77aa6657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732664070842 2024-11-26T23:34:32,127 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5301f81964215ffd1194b5e2f86d347#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:32,128 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/214d544c0bb34f60b59496228a9b5d2b is 50, key is test_row_0/C:col10/1732664070845/Put/seqid=0 2024-11-26T23:34:32,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742135_1311 (size=13221) 2024-11-26T23:34:32,517 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/b330eed277c740b78784fe44da2acf23 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b330eed277c740b78784fe44da2acf23 2024-11-26T23:34:32,520 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/B of c5301f81964215ffd1194b5e2f86d347 into b330eed277c740b78784fe44da2acf23(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:32,520 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:32,520 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/B, priority=13, startTime=1732664072100; duration=0sec 2024-11-26T23:34:32,521 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:32,521 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:B 2024-11-26T23:34:32,535 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/214d544c0bb34f60b59496228a9b5d2b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/214d544c0bb34f60b59496228a9b5d2b 2024-11-26T23:34:32,538 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5301f81964215ffd1194b5e2f86d347/C of c5301f81964215ffd1194b5e2f86d347 into 214d544c0bb34f60b59496228a9b5d2b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:32,539 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:32,539 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., storeName=c5301f81964215ffd1194b5e2f86d347/C, priority=13, startTime=1732664072100; duration=0sec 2024-11-26T23:34:32,539 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:32,539 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5301f81964215ffd1194b5e2f86d347:C 2024-11-26T23:34:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T23:34:32,752 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:32,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:32,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:34:32,903 INFO [Thread-1064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1064. 2024-11-26T23:34:32,903 DEBUG [Thread-1064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:32,903 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:32,903 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:32,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:32,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/cb7f5a03116f440badeca8c64d5a48bd is 50, key is test_row_0/A:col10/1732664070884/Put/seqid=0 2024-11-26T23:34:32,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742136_1312 (size=12301) 2024-11-26T23:34:33,033 INFO [Thread-1066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1066. 2024-11-26T23:34:33,033 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-26T23:34:33,033 DEBUG [Thread-1066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:33,033 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:33,033 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,033 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,033 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,033 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:33,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:33,043 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-26T23:34:33,043 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:33,043 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,043 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:33,052 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-26T23:34:33,052 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:34:33,052 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,052 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3763 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3679 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3691 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3739 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3709 2024-11-26T23:34:33,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T23:34:33,053 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:34:33,053 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:34:33,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:33,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:34:33,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T23:34:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T23:34:33,056 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664073056"}]},"ts":"1732664073056"} 2024-11-26T23:34:33,058 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T23:34:33,058 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T23:34:33,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:34:33,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, UNASSIGN}] 2024-11-26T23:34:33,060 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, UNASSIGN 2024-11-26T23:34:33,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c5301f81964215ffd1194b5e2f86d347, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:33,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, UNASSIGN because future has completed 2024-11-26T23:34:33,061 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:34:33,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:34:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T23:34:33,213 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:33,213 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:34:33,213 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing c5301f81964215ffd1194b5e2f86d347, disabling compactions & flushes 2024-11-26T23:34:33,213 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:33,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/cb7f5a03116f440badeca8c64d5a48bd 2024-11-26T23:34:33,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/e63bf5fa0065422f8be670a72ff6a9c3 is 50, key is test_row_0/B:col10/1732664070884/Put/seqid=0 2024-11-26T23:34:33,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742137_1313 (size=12301) 2024-11-26T23:34:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T23:34:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T23:34:33,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/e63bf5fa0065422f8be670a72ff6a9c3 2024-11-26T23:34:33,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9ec76b03d537419899aba826b839adb8 is 50, key is test_row_0/C:col10/1732664070884/Put/seqid=0 2024-11-26T23:34:33,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742138_1314 (size=12301) 2024-11-26T23:34:34,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9ec76b03d537419899aba826b839adb8 2024-11-26T23:34:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/cb7f5a03116f440badeca8c64d5a48bd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cb7f5a03116f440badeca8c64d5a48bd 2024-11-26T23:34:34,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cb7f5a03116f440badeca8c64d5a48bd, entries=150, sequenceid=411, filesize=12.0 K 2024-11-26T23:34:34,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/e63bf5fa0065422f8be670a72ff6a9c3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e63bf5fa0065422f8be670a72ff6a9c3 2024-11-26T23:34:34,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e63bf5fa0065422f8be670a72ff6a9c3, entries=150, sequenceid=411, filesize=12.0 K 2024-11-26T23:34:34,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/9ec76b03d537419899aba826b839adb8 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec76b03d537419899aba826b839adb8 2024-11-26T23:34:34,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec76b03d537419899aba826b839adb8, entries=150, sequenceid=411, filesize=12.0 K 2024-11-26T23:34:34,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=26.84 KB/27480 for c5301f81964215ffd1194b5e2f86d347 in 1243ms, sequenceid=411, compaction requested=false 2024-11-26T23:34:34,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5301f81964215ffd1194b5e2f86d347: 2024-11-26T23:34:34,146 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:34,146 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:34,146 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. after waiting 0 ms 2024-11-26T23:34:34,146 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:34,146 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing c5301f81964215ffd1194b5e2f86d347 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-26T23:34:34,149 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/38aef1375b6648fe9fd5d436669a51cb is 50, key is test_row_0/A:col10/1732664073051/Put/seqid=0 2024-11-26T23:34:34,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742139_1315 (size=12301) 2024-11-26T23:34:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T23:34:34,553 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/38aef1375b6648fe9fd5d436669a51cb 2024-11-26T23:34:34,559 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c2858d467b7141e28d3478b2ae0785b5 is 50, key is test_row_0/B:col10/1732664073051/Put/seqid=0 2024-11-26T23:34:34,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742140_1316 (size=12301) 2024-11-26T23:34:34,597 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:34:34,963 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c2858d467b7141e28d3478b2ae0785b5 2024-11-26T23:34:34,969 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/6f7f6aac8a4b4436b205afe995080d74 is 50, key is test_row_0/C:col10/1732664073051/Put/seqid=0 2024-11-26T23:34:34,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742141_1317 (size=12301) 2024-11-26T23:34:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T23:34:35,373 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/6f7f6aac8a4b4436b205afe995080d74 2024-11-26T23:34:35,377 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/A/38aef1375b6648fe9fd5d436669a51cb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/38aef1375b6648fe9fd5d436669a51cb 2024-11-26T23:34:35,381 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/38aef1375b6648fe9fd5d436669a51cb, entries=150, sequenceid=418, filesize=12.0 K 2024-11-26T23:34:35,382 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/B/c2858d467b7141e28d3478b2ae0785b5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2858d467b7141e28d3478b2ae0785b5 2024-11-26T23:34:35,385 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2858d467b7141e28d3478b2ae0785b5, entries=150, sequenceid=418, filesize=12.0 K 2024-11-26T23:34:35,385 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/.tmp/C/6f7f6aac8a4b4436b205afe995080d74 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6f7f6aac8a4b4436b205afe995080d74 2024-11-26T23:34:35,389 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6f7f6aac8a4b4436b205afe995080d74, entries=150, sequenceid=418, filesize=12.0 K 2024-11-26T23:34:35,390 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c5301f81964215ffd1194b5e2f86d347 in 1244ms, sequenceid=418, compaction requested=true 2024-11-26T23:34:35,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/4ddb75a354124179a30fae6c19a41bdc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/a8fd74c10886444db53616a44fc39259, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/6c066bb7f2744c6b9b2b19526038b5c5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/5add2a0f3f244ef1aa50931607516b11, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cc2656c5371c4e5495eb18d636626fd1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/aad558e122ff4cf6ab8fd6cb9bdde0d3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2e5b10410a6a43cab23102a341d76bc4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/9bffba8dbf1b4dfca05dbfc995130698, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0e97e23ef33549bfa2f7362c0467efe2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0af4a2f5592a4be09997823557e0414c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f023d930e42949d1809771e9083a57b5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/49066b0ee43547b9884325c449304405, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/232acf976bc54b44b31f6060e29b986e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0fb1f97c55cd483d844c9cf40dfe8c80, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/fc25103ec80b4c54b486949535e98647, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7005fde2595246038adb877673efde74, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/713040a469c94b24b0a784d7a61d97b7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/3191e584c3cd43ac91a626cf8ba3ec33, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/67b7677ef7cc43008972a0df47e9daab, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7e7aeda104264005bd58716c2f63dd0e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/10fd5c3c9a3248699d8ccacf6fde50b6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2c91675c429c4f638e58d7e60f7c0eba, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/89b8e50a23ec4b58b147fb67810c6484, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2aad309d028d4746ae95d48d03ca58a9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ad7d5c908af0400b8a5f9fd57adf4e58, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ee880bf6577d483b8bbcaa5242c97614, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/258b1e49d62541ddb064aa8dced40baf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f7d1da867adb4527b27274b127346434] to archive 2024-11-26T23:34:35,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:34:35,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/4ddb75a354124179a30fae6c19a41bdc to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/4ddb75a354124179a30fae6c19a41bdc 2024-11-26T23:34:35,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/a8fd74c10886444db53616a44fc39259 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/a8fd74c10886444db53616a44fc39259 2024-11-26T23:34:35,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/6c066bb7f2744c6b9b2b19526038b5c5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/6c066bb7f2744c6b9b2b19526038b5c5 2024-11-26T23:34:35,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/5add2a0f3f244ef1aa50931607516b11 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/5add2a0f3f244ef1aa50931607516b11 2024-11-26T23:34:35,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cc2656c5371c4e5495eb18d636626fd1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cc2656c5371c4e5495eb18d636626fd1 2024-11-26T23:34:35,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/aad558e122ff4cf6ab8fd6cb9bdde0d3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/aad558e122ff4cf6ab8fd6cb9bdde0d3 2024-11-26T23:34:35,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2e5b10410a6a43cab23102a341d76bc4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2e5b10410a6a43cab23102a341d76bc4 2024-11-26T23:34:35,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/9bffba8dbf1b4dfca05dbfc995130698 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/9bffba8dbf1b4dfca05dbfc995130698 2024-11-26T23:34:35,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0e97e23ef33549bfa2f7362c0467efe2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0e97e23ef33549bfa2f7362c0467efe2 2024-11-26T23:34:35,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0af4a2f5592a4be09997823557e0414c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0af4a2f5592a4be09997823557e0414c 2024-11-26T23:34:35,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f023d930e42949d1809771e9083a57b5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f023d930e42949d1809771e9083a57b5 2024-11-26T23:34:35,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/49066b0ee43547b9884325c449304405 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/49066b0ee43547b9884325c449304405 2024-11-26T23:34:35,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/232acf976bc54b44b31f6060e29b986e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/232acf976bc54b44b31f6060e29b986e 2024-11-26T23:34:35,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0fb1f97c55cd483d844c9cf40dfe8c80 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0fb1f97c55cd483d844c9cf40dfe8c80 2024-11-26T23:34:35,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/fc25103ec80b4c54b486949535e98647 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/fc25103ec80b4c54b486949535e98647 2024-11-26T23:34:35,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7005fde2595246038adb877673efde74 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7005fde2595246038adb877673efde74 2024-11-26T23:34:35,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/713040a469c94b24b0a784d7a61d97b7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/713040a469c94b24b0a784d7a61d97b7 2024-11-26T23:34:35,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/3191e584c3cd43ac91a626cf8ba3ec33 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/3191e584c3cd43ac91a626cf8ba3ec33 2024-11-26T23:34:35,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/67b7677ef7cc43008972a0df47e9daab to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/67b7677ef7cc43008972a0df47e9daab 2024-11-26T23:34:35,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7e7aeda104264005bd58716c2f63dd0e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/7e7aeda104264005bd58716c2f63dd0e 2024-11-26T23:34:35,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/10fd5c3c9a3248699d8ccacf6fde50b6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/10fd5c3c9a3248699d8ccacf6fde50b6 2024-11-26T23:34:35,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2c91675c429c4f638e58d7e60f7c0eba to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2c91675c429c4f638e58d7e60f7c0eba 2024-11-26T23:34:35,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/89b8e50a23ec4b58b147fb67810c6484 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/89b8e50a23ec4b58b147fb67810c6484 2024-11-26T23:34:35,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2aad309d028d4746ae95d48d03ca58a9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/2aad309d028d4746ae95d48d03ca58a9 2024-11-26T23:34:35,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ad7d5c908af0400b8a5f9fd57adf4e58 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ad7d5c908af0400b8a5f9fd57adf4e58 2024-11-26T23:34:35,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ee880bf6577d483b8bbcaa5242c97614 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/ee880bf6577d483b8bbcaa5242c97614 2024-11-26T23:34:35,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/258b1e49d62541ddb064aa8dced40baf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/258b1e49d62541ddb064aa8dced40baf 2024-11-26T23:34:35,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f7d1da867adb4527b27274b127346434 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/f7d1da867adb4527b27274b127346434 2024-11-26T23:34:35,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/0496a22283064e229ab1e49a199bcdaf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e92ee4769a6e4f328daf023fc1ec3dec, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b97ff1aae7364a4ba9bc3c10bad4b594, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/35bf09f1adb6468a8c530535bc58211b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4c078b74ae44474fb3713d518b80fecc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/bbe93b8250ce43c1a737235ecbef953a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c6a59340183942dfb31083d1cf34f86a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b7dc2ef041914dbaa061b28ba9622869, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8447232c649f40ef8a8245cd594c5ec2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4705531fb5d24a2c9e52ef48fbd842b0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c1c5999dff9c434eb5f2ee3fdeaec83f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/aacad87609c14d5aa21734fef33dc6b4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/1cc03dd4144f4212b2cf5338b27e8acd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/375f544c19af4d93956f49adda1694cb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/7279b69ac2074068beb73acf9fb12a0d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b45bd2d6457249d881a40d9d2cb46584, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d2dbaec4cf80416e84f94b6588c19a00, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b9bf04a4617f442aa625fbd3621b9c7f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5a97b58b687c46fdb85fa93ef49c54cf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d18aefea7354402da951f59a5bfd70a5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2fa3bc1cfc340cebbb6573db8eedce0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8454315e8555403997813028e7c45e72, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/addd667d1efa4aae9777a29857b67a97, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/6e37634ad1244391bf2cd91b61e2b8be, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5d78afacd32c47fdae73dff10d5289ee, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/478268e4bc14471793bfbd327b51d8d2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8daaf07141db41b9aeaaf5f88c675981, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/10b66dfba66a4fc4a5adf64ceda79596] to archive 2024-11-26T23:34:35,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:34:35,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/0496a22283064e229ab1e49a199bcdaf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/0496a22283064e229ab1e49a199bcdaf 2024-11-26T23:34:35,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e92ee4769a6e4f328daf023fc1ec3dec to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e92ee4769a6e4f328daf023fc1ec3dec 2024-11-26T23:34:35,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b97ff1aae7364a4ba9bc3c10bad4b594 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b97ff1aae7364a4ba9bc3c10bad4b594 2024-11-26T23:34:35,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/35bf09f1adb6468a8c530535bc58211b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/35bf09f1adb6468a8c530535bc58211b 2024-11-26T23:34:35,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4c078b74ae44474fb3713d518b80fecc to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4c078b74ae44474fb3713d518b80fecc 2024-11-26T23:34:35,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/bbe93b8250ce43c1a737235ecbef953a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/bbe93b8250ce43c1a737235ecbef953a 2024-11-26T23:34:35,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c6a59340183942dfb31083d1cf34f86a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c6a59340183942dfb31083d1cf34f86a 2024-11-26T23:34:35,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b7dc2ef041914dbaa061b28ba9622869 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b7dc2ef041914dbaa061b28ba9622869 2024-11-26T23:34:35,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8447232c649f40ef8a8245cd594c5ec2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8447232c649f40ef8a8245cd594c5ec2 2024-11-26T23:34:35,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4705531fb5d24a2c9e52ef48fbd842b0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/4705531fb5d24a2c9e52ef48fbd842b0 2024-11-26T23:34:35,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c1c5999dff9c434eb5f2ee3fdeaec83f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c1c5999dff9c434eb5f2ee3fdeaec83f 2024-11-26T23:34:35,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/aacad87609c14d5aa21734fef33dc6b4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/aacad87609c14d5aa21734fef33dc6b4 2024-11-26T23:34:35,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/1cc03dd4144f4212b2cf5338b27e8acd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/1cc03dd4144f4212b2cf5338b27e8acd 2024-11-26T23:34:35,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/375f544c19af4d93956f49adda1694cb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/375f544c19af4d93956f49adda1694cb 2024-11-26T23:34:35,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/7279b69ac2074068beb73acf9fb12a0d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/7279b69ac2074068beb73acf9fb12a0d 2024-11-26T23:34:35,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b45bd2d6457249d881a40d9d2cb46584 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b45bd2d6457249d881a40d9d2cb46584 2024-11-26T23:34:35,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d2dbaec4cf80416e84f94b6588c19a00 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d2dbaec4cf80416e84f94b6588c19a00 2024-11-26T23:34:35,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b9bf04a4617f442aa625fbd3621b9c7f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b9bf04a4617f442aa625fbd3621b9c7f 2024-11-26T23:34:35,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5a97b58b687c46fdb85fa93ef49c54cf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5a97b58b687c46fdb85fa93ef49c54cf 2024-11-26T23:34:35,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d18aefea7354402da951f59a5bfd70a5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/d18aefea7354402da951f59a5bfd70a5 2024-11-26T23:34:35,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2fa3bc1cfc340cebbb6573db8eedce0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2fa3bc1cfc340cebbb6573db8eedce0 2024-11-26T23:34:35,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8454315e8555403997813028e7c45e72 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8454315e8555403997813028e7c45e72 2024-11-26T23:34:35,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/addd667d1efa4aae9777a29857b67a97 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/addd667d1efa4aae9777a29857b67a97 2024-11-26T23:34:35,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/6e37634ad1244391bf2cd91b61e2b8be to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/6e37634ad1244391bf2cd91b61e2b8be 2024-11-26T23:34:35,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5d78afacd32c47fdae73dff10d5289ee to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/5d78afacd32c47fdae73dff10d5289ee 2024-11-26T23:34:35,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/478268e4bc14471793bfbd327b51d8d2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/478268e4bc14471793bfbd327b51d8d2 2024-11-26T23:34:35,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8daaf07141db41b9aeaaf5f88c675981 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/8daaf07141db41b9aeaaf5f88c675981 2024-11-26T23:34:35,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/10b66dfba66a4fc4a5adf64ceda79596 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/10b66dfba66a4fc4a5adf64ceda79596 2024-11-26T23:34:35,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/cf386b5fa760412aaffa14dd4eed42a7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/468c9eece52945a08a2d335d0750031b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/374d539db06c4ea98c03e3f0d35cef5b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9bba5415f8894d6fa99c647bae0eb5bd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/ef91ffc4329d4f96812633d252a19eb9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec69f0371fd49feadb27d5bd1a2535a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c4ba11f5cbfd42539cd6d2d7d7092564, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/a2222e5beea44f1cbd39dd6802fc8461, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/083e7a2f034c4b82a81b2d6aea508c0f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/d0ba524de75c401dad0a8fe480bc56b3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/78247c03b5a642009d9e8238b213f356, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9352264c416943a9949e95466ba21458, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/268776cf0232464eafd281e006f52dc7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f9b74dc9536b4105b2b41a69737b6fd1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/237d09eadc1642aa900d6611d412e2f1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6531e3bb032c465a8d0f57489d7974c5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/4f8ec8918fc947caae9fd92962c320fd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f0480f2bb826488e992a5296f33b6e49, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1c506437ec9247ca8e8976689bdff17c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/7917682e0de748c08acd744f298472e7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c11bd46ab8214285a9714901d45ce07f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/64c96feeebf146a2b28f08b49fbb9f52, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/326869def9214a94aab97e4c2b87bb89, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1e36888305714c749d7692e497ac7cc7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/2932c609067741ce9f40a59073ea2ade, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/b2ff539073044ddf8db1b1d5fe905c63, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/e995cfb0696d46ef871a8a7b77aa6657] to archive 2024-11-26T23:34:35,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:34:35,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/cf386b5fa760412aaffa14dd4eed42a7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/cf386b5fa760412aaffa14dd4eed42a7 2024-11-26T23:34:35,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/468c9eece52945a08a2d335d0750031b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/468c9eece52945a08a2d335d0750031b 2024-11-26T23:34:35,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/374d539db06c4ea98c03e3f0d35cef5b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/374d539db06c4ea98c03e3f0d35cef5b 2024-11-26T23:34:35,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9bba5415f8894d6fa99c647bae0eb5bd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9bba5415f8894d6fa99c647bae0eb5bd 2024-11-26T23:34:35,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/ef91ffc4329d4f96812633d252a19eb9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/ef91ffc4329d4f96812633d252a19eb9 2024-11-26T23:34:35,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec69f0371fd49feadb27d5bd1a2535a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec69f0371fd49feadb27d5bd1a2535a 2024-11-26T23:34:35,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c4ba11f5cbfd42539cd6d2d7d7092564 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c4ba11f5cbfd42539cd6d2d7d7092564 2024-11-26T23:34:35,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/a2222e5beea44f1cbd39dd6802fc8461 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/a2222e5beea44f1cbd39dd6802fc8461 2024-11-26T23:34:35,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/083e7a2f034c4b82a81b2d6aea508c0f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/083e7a2f034c4b82a81b2d6aea508c0f 2024-11-26T23:34:35,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/d0ba524de75c401dad0a8fe480bc56b3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/d0ba524de75c401dad0a8fe480bc56b3 2024-11-26T23:34:35,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/78247c03b5a642009d9e8238b213f356 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/78247c03b5a642009d9e8238b213f356 2024-11-26T23:34:35,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9352264c416943a9949e95466ba21458 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9352264c416943a9949e95466ba21458 2024-11-26T23:34:35,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/268776cf0232464eafd281e006f52dc7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/268776cf0232464eafd281e006f52dc7 2024-11-26T23:34:35,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f9b74dc9536b4105b2b41a69737b6fd1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f9b74dc9536b4105b2b41a69737b6fd1 2024-11-26T23:34:35,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/237d09eadc1642aa900d6611d412e2f1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/237d09eadc1642aa900d6611d412e2f1 2024-11-26T23:34:35,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6531e3bb032c465a8d0f57489d7974c5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6531e3bb032c465a8d0f57489d7974c5 2024-11-26T23:34:35,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/4f8ec8918fc947caae9fd92962c320fd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/4f8ec8918fc947caae9fd92962c320fd 2024-11-26T23:34:35,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f0480f2bb826488e992a5296f33b6e49 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/f0480f2bb826488e992a5296f33b6e49 2024-11-26T23:34:35,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1c506437ec9247ca8e8976689bdff17c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1c506437ec9247ca8e8976689bdff17c 2024-11-26T23:34:35,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/7917682e0de748c08acd744f298472e7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/7917682e0de748c08acd744f298472e7 2024-11-26T23:34:35,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c11bd46ab8214285a9714901d45ce07f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/c11bd46ab8214285a9714901d45ce07f 2024-11-26T23:34:35,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/64c96feeebf146a2b28f08b49fbb9f52 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/64c96feeebf146a2b28f08b49fbb9f52 2024-11-26T23:34:35,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/326869def9214a94aab97e4c2b87bb89 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/326869def9214a94aab97e4c2b87bb89 2024-11-26T23:34:35,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1e36888305714c749d7692e497ac7cc7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/1e36888305714c749d7692e497ac7cc7 2024-11-26T23:34:35,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/2932c609067741ce9f40a59073ea2ade to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/2932c609067741ce9f40a59073ea2ade 2024-11-26T23:34:35,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/b2ff539073044ddf8db1b1d5fe905c63 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/b2ff539073044ddf8db1b1d5fe905c63 2024-11-26T23:34:35,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/e995cfb0696d46ef871a8a7b77aa6657 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/e995cfb0696d46ef871a8a7b77aa6657 2024-11-26T23:34:35,474 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/recovered.edits/421.seqid, newMaxSeqId=421, maxSeqId=1 2024-11-26T23:34:35,475 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. 2024-11-26T23:34:35,475 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for c5301f81964215ffd1194b5e2f86d347: Waiting for close lock at 1732664073213Running coprocessor pre-close hooks at 1732664073213Disabling compacts and flushes for region at 1732664073213Disabling writes for close at 1732664074146 (+933 ms)Obtaining lock to block concurrent updates at 1732664074146Preparing flush snapshotting stores in c5301f81964215ffd1194b5e2f86d347 at 1732664074146Finished memstore snapshotting TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732664074146Flushing stores of TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347. at 1732664074146Flushing c5301f81964215ffd1194b5e2f86d347/A: creating writer at 1732664074146Flushing c5301f81964215ffd1194b5e2f86d347/A: appending metadata at 1732664074149 (+3 ms)Flushing c5301f81964215ffd1194b5e2f86d347/A: closing flushed file at 1732664074149Flushing c5301f81964215ffd1194b5e2f86d347/B: creating writer at 1732664074556 (+407 ms)Flushing c5301f81964215ffd1194b5e2f86d347/B: appending metadata at 1732664074558 (+2 ms)Flushing c5301f81964215ffd1194b5e2f86d347/B: closing flushed file at 1732664074558Flushing c5301f81964215ffd1194b5e2f86d347/C: creating writer at 1732664074966 (+408 ms)Flushing c5301f81964215ffd1194b5e2f86d347/C: appending metadata at 1732664074969 (+3 ms)Flushing c5301f81964215ffd1194b5e2f86d347/C: closing flushed file at 1732664074969Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@672c3c1d: reopening flushed file at 1732664075377 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a31f014: reopening flushed file at 1732664075381 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33dbf0db: reopening flushed file at 1732664075385 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c5301f81964215ffd1194b5e2f86d347 in 1244ms, sequenceid=418, compaction requested=true at 1732664075390 (+5 ms)Writing region close event to WAL at 1732664075471 (+81 ms)Running coprocessor post-close hooks at 1732664075475 (+4 ms)Closed at 1732664075475 2024-11-26T23:34:35,477 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:35,477 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c5301f81964215ffd1194b5e2f86d347, regionState=CLOSED 2024-11-26T23:34:35,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:34:35,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-26T23:34:35,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure c5301f81964215ffd1194b5e2f86d347, server=c707e9ace675,43077,1732663986534 in 2.4180 sec 2024-11-26T23:34:35,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-26T23:34:35,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5301f81964215ffd1194b5e2f86d347, UNASSIGN in 2.4220 sec 2024-11-26T23:34:35,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-26T23:34:35,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4250 sec 2024-11-26T23:34:35,486 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664075485"}]},"ts":"1732664075485"} 2024-11-26T23:34:35,487 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T23:34:35,487 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T23:34:35,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4330 sec 2024-11-26T23:34:36,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T23:34:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T23:34:37,212 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:37,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T23:34:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:37,214 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T23:34:37,215 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:37,216 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:37,218 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/recovered.edits] 2024-11-26T23:34:37,220 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0432443df7f949a3ad9b2ed0484fd9cc to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/0432443df7f949a3ad9b2ed0484fd9cc 2024-11-26T23:34:37,221 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/38aef1375b6648fe9fd5d436669a51cb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/38aef1375b6648fe9fd5d436669a51cb 2024-11-26T23:34:37,222 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cb7f5a03116f440badeca8c64d5a48bd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/A/cb7f5a03116f440badeca8c64d5a48bd 2024-11-26T23:34:37,225 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b330eed277c740b78784fe44da2acf23 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/b330eed277c740b78784fe44da2acf23 2024-11-26T23:34:37,226 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2858d467b7141e28d3478b2ae0785b5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/c2858d467b7141e28d3478b2ae0785b5 2024-11-26T23:34:37,227 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e63bf5fa0065422f8be670a72ff6a9c3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/B/e63bf5fa0065422f8be670a72ff6a9c3 2024-11-26T23:34:37,228 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/214d544c0bb34f60b59496228a9b5d2b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/214d544c0bb34f60b59496228a9b5d2b 2024-11-26T23:34:37,229 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6f7f6aac8a4b4436b205afe995080d74 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/6f7f6aac8a4b4436b205afe995080d74 2024-11-26T23:34:37,230 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec76b03d537419899aba826b839adb8 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/C/9ec76b03d537419899aba826b839adb8 2024-11-26T23:34:37,232 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/recovered.edits/421.seqid to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347/recovered.edits/421.seqid 2024-11-26T23:34:37,233 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/c5301f81964215ffd1194b5e2f86d347 2024-11-26T23:34:37,233 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T23:34:37,235 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:37,238 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T23:34:37,240 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T23:34:37,241 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:37,241 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T23:34:37,241 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732664077241"}]},"ts":"9223372036854775807"} 2024-11-26T23:34:37,242 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T23:34:37,242 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c5301f81964215ffd1194b5e2f86d347, NAME => 'TestAcidGuarantees,,1732664048690.c5301f81964215ffd1194b5e2f86d347.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T23:34:37,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T23:34:37,243 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732664077242"}]},"ts":"9223372036854775807"} 2024-11-26T23:34:37,244 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T23:34:37,245 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:37,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-26T23:34:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T23:34:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T23:34:37,322 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:37,331 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1770722268_22 at /127.0.0.1:42156 [Waiting for operation #638] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_560703445_22 at /127.0.0.1:42178 [Waiting for operation #621] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 346) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6540 (was 6733) 2024-11-26T23:34:37,339 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=6540 2024-11-26T23:34:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:34:37,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:34:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:37,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:34:37,343 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:37,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-26T23:34:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T23:34:37,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:34:37,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742142_1318 (size=959) 2024-11-26T23:34:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T23:34:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T23:34:37,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:34:37,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742143_1319 (size=53) 2024-11-26T23:34:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T23:34:38,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:34:38,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 38491e1fe242cb24ed1e0e85404fb0f7, disabling compactions & flushes 2024-11-26T23:34:38,156 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:38,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:38,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. after waiting 0 ms 2024-11-26T23:34:38,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:38,156 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:38,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38491e1fe242cb24ed1e0e85404fb0f7: Waiting for close lock at 1732664078156Disabling compacts and flushes for region at 1732664078156Disabling writes for close at 1732664078156Writing region close event to WAL at 1732664078156Closed at 1732664078156 2024-11-26T23:34:38,157 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:34:38,158 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732664078157"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732664078157"}]},"ts":"1732664078157"} 2024-11-26T23:34:38,159 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:34:38,160 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:34:38,160 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664078160"}]},"ts":"1732664078160"} 2024-11-26T23:34:38,161 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T23:34:38,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, ASSIGN}] 2024-11-26T23:34:38,162 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, ASSIGN 2024-11-26T23:34:38,163 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, ASSIGN; state=OFFLINE, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=false 2024-11-26T23:34:38,313 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:38,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, ASSIGN because future has completed 2024-11-26T23:34:38,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:34:38,470 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:38,470 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:34:38,470 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,470 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:34:38,470 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,470 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,471 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,473 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38491e1fe242cb24ed1e0e85404fb0f7 columnFamilyName A 2024-11-26T23:34:38,473 DEBUG [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:38,473 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(327): Store=38491e1fe242cb24ed1e0e85404fb0f7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:38,473 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,474 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38491e1fe242cb24ed1e0e85404fb0f7 columnFamilyName B 2024-11-26T23:34:38,474 DEBUG [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:38,474 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(327): Store=38491e1fe242cb24ed1e0e85404fb0f7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:38,474 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,475 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38491e1fe242cb24ed1e0e85404fb0f7 columnFamilyName C 2024-11-26T23:34:38,475 DEBUG [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:38,475 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(327): Store=38491e1fe242cb24ed1e0e85404fb0f7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:38,476 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,476 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,476 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,477 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,477 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,477 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:34:38,478 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,480 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:34:38,480 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 38491e1fe242cb24ed1e0e85404fb0f7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66349737, jitterRate=-0.01131187379360199}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:34:38,480 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:38,481 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 38491e1fe242cb24ed1e0e85404fb0f7: Running coprocessor pre-open hook at 1732664078470Writing region info on filesystem at 1732664078470Initializing all the Stores at 1732664078471 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664078471Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664078471Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664078471Cleaning up temporary data from old regions at 1732664078477 (+6 ms)Running coprocessor post-open hooks at 1732664078480 (+3 ms)Region opened successfully at 1732664078481 (+1 ms) 2024-11-26T23:34:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T23:34:38,481 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., pid=87, masterSystemTime=1732664078466 2024-11-26T23:34:38,483 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:38,483 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:38,484 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:38,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:34:38,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-26T23:34:38,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 in 171 msec 2024-11-26T23:34:38,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-26T23:34:38,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, ASSIGN in 326 msec 2024-11-26T23:34:38,489 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:34:38,489 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664078489"}]},"ts":"1732664078489"} 2024-11-26T23:34:38,490 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T23:34:38,490 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:34:38,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-26T23:34:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T23:34:39,492 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:39,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:39,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:39,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:39,494 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:39,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:39,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:39,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784592c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:39,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:39,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:39,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:39,496 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4047087f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:39,496 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:39,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:39,499 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:34:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35833,1732663985624 2024-11-26T23:34:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79a8c0c3 2024-11-26T23:34:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:34:39,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:34:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:34:39,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:34:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T23:34:39,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742144_1320 (size=995) 2024-11-26T23:34:39,913 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-26T23:34:39,913 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-26T23:34:39,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:34:39,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, REOPEN/MOVE}] 2024-11-26T23:34:39,918 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, REOPEN/MOVE 2024-11-26T23:34:39,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:39,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, REOPEN/MOVE because future has completed 2024-11-26T23:34:39,920 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:34:39,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:34:40,072 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,072 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:34:40,072 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 38491e1fe242cb24ed1e0e85404fb0f7, disabling compactions & flushes 2024-11-26T23:34:40,072 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,072 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,072 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. after waiting 0 ms 2024-11-26T23:34:40,072 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,075 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T23:34:40,076 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,076 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 38491e1fe242cb24ed1e0e85404fb0f7: Waiting for close lock at 1732664080072Running coprocessor pre-close hooks at 1732664080072Disabling compacts and flushes for region at 1732664080072Disabling writes for close at 1732664080072Writing region close event to WAL at 1732664080073 (+1 ms)Running coprocessor post-close hooks at 1732664080076 (+3 ms)Closed at 1732664080076 2024-11-26T23:34:40,076 WARN [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 38491e1fe242cb24ed1e0e85404fb0f7 to self. 2024-11-26T23:34:40,077 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,078 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=CLOSED 2024-11-26T23:34:40,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:34:40,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-26T23:34:40,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 in 160 msec 2024-11-26T23:34:40,081 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, REOPEN/MOVE; state=CLOSED, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=true 2024-11-26T23:34:40,232 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, REOPEN/MOVE because future has completed 2024-11-26T23:34:40,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:34:40,388 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,388 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:34:40,388 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,389 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:34:40,389 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,389 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,390 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,390 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38491e1fe242cb24ed1e0e85404fb0f7 columnFamilyName A 2024-11-26T23:34:40,391 DEBUG [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:40,392 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(327): Store=38491e1fe242cb24ed1e0e85404fb0f7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:40,392 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,393 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38491e1fe242cb24ed1e0e85404fb0f7 columnFamilyName B 2024-11-26T23:34:40,393 DEBUG [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:40,393 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(327): Store=38491e1fe242cb24ed1e0e85404fb0f7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:40,393 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,394 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38491e1fe242cb24ed1e0e85404fb0f7 columnFamilyName C 2024-11-26T23:34:40,394 DEBUG [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:40,394 INFO [StoreOpener-38491e1fe242cb24ed1e0e85404fb0f7-1 {}] regionserver.HStore(327): Store=38491e1fe242cb24ed1e0e85404fb0f7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:34:40,394 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,394 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,395 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,396 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,396 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,396 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:34:40,397 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,398 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 38491e1fe242cb24ed1e0e85404fb0f7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60030134, jitterRate=-0.10548129677772522}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:34:40,398 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,398 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 38491e1fe242cb24ed1e0e85404fb0f7: Running coprocessor pre-open hook at 1732664080389Writing region info on filesystem at 1732664080389Initializing all the Stores at 1732664080389Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664080389Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664080390 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664080390Cleaning up temporary data from old regions at 1732664080396 (+6 ms)Running coprocessor post-open hooks at 1732664080398 (+2 ms)Region opened successfully at 1732664080398 2024-11-26T23:34:40,399 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., pid=92, masterSystemTime=1732664080385 2024-11-26T23:34:40,400 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,400 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,401 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=OPEN, openSeqNum=5, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:34:40,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-26T23:34:40,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 in 169 msec 2024-11-26T23:34:40,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-26T23:34:40,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, REOPEN/MOVE in 487 msec 2024-11-26T23:34:40,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-26T23:34:40,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-26T23:34:40,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-26T23:34:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T23:34:40,410 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:40,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f55d0a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f302df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eceff4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4ff04d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,418 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a5cda4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,420 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b728ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,421 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe3345d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,422 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f36f46e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e3c247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8e336e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3be3a1ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,428 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424346e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e24be6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,430 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66813f06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b162f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,433 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459d160d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d148972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:34:40,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:34:40,437 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:34:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:34:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:34:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1618da56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:34:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:34:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:34:40,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:34:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee3ece1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:34:40,439 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,439 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,439 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,439 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,439 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,439 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:40,439 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,440 DEBUG [Thread-1474 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,440 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,440 DEBUG [Thread-1474 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,440 DEBUG [Thread-1477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-26T23:34:40,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,441 DEBUG [Thread-1477 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T23:34:40,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,442 DEBUG [Thread-1477 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,444 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:40,444 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,445 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:40,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,457 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:34:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:34:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:34:40,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:34:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:34:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:40,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:40,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126afe0dcd103a04a16be26c952c490aaad_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664080461/Put/seqid=0 2024-11-26T23:34:40,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742145_1321 (size=14594) 2024-11-26T23:34:40,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664140499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664140504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664140508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664140510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664140510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T23:34:40,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:40,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:40,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664140621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664140621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664140631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664140631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664140632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T23:34:40,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664140831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664140831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664140852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664140852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664140852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:40,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:40,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:40,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:40,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:40,910 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:40,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126afe0dcd103a04a16be26c952c490aaad_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126afe0dcd103a04a16be26c952c490aaad_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:40,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/704ccf06f89a4b76ac2c4fca4f572b71, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:40,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/704ccf06f89a4b76ac2c4fca4f572b71 is 175, key is test_row_0/A:col10/1732664080461/Put/seqid=0 2024-11-26T23:34:40,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742146_1322 (size=39549) 2024-11-26T23:34:40,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/704ccf06f89a4b76ac2c4fca4f572b71 2024-11-26T23:34:40,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/0fc1251dbc1f4045bf03daaf440001c6 is 50, key is test_row_0/B:col10/1732664080461/Put/seqid=0 2024-11-26T23:34:40,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742147_1323 (size=12001) 2024-11-26T23:34:41,002 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:34:41,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:41,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:41,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T23:34:41,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664141141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664141152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664141162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664141162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664141162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:41,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:41,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/0fc1251dbc1f4045bf03daaf440001c6 2024-11-26T23:34:41,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:41,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:41,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/2854896927a74dc49775230c121e9090 is 50, key is test_row_0/C:col10/1732664080461/Put/seqid=0 2024-11-26T23:34:41,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742148_1324 (size=12001) 2024-11-26T23:34:41,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:41,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:41,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T23:34:41,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664141662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664141662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:41,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:41,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:41,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664141681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664141681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664141681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:41,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/2854896927a74dc49775230c121e9090 2024-11-26T23:34:41,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/704ccf06f89a4b76ac2c4fca4f572b71 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/704ccf06f89a4b76ac2c4fca4f572b71 2024-11-26T23:34:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/704ccf06f89a4b76ac2c4fca4f572b71, entries=200, sequenceid=18, filesize=38.6 K 2024-11-26T23:34:41,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/0fc1251dbc1f4045bf03daaf440001c6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/0fc1251dbc1f4045bf03daaf440001c6 2024-11-26T23:34:41,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/0fc1251dbc1f4045bf03daaf440001c6, entries=150, sequenceid=18, filesize=11.7 K 2024-11-26T23:34:41,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/2854896927a74dc49775230c121e9090 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2854896927a74dc49775230c121e9090 2024-11-26T23:34:41,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2854896927a74dc49775230c121e9090, entries=150, sequenceid=18, filesize=11.7 K 2024-11-26T23:34:41,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1339ms, sequenceid=18, compaction requested=false 2024-11-26T23:34:41,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:41,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T23:34:41,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:41,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T23:34:41,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fe20c1cf817a4d0a86be7494cc3103eb_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664080508/Put/seqid=0 2024-11-26T23:34:41,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742149_1325 (size=12154) 2024-11-26T23:34:42,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:42,245 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fe20c1cf817a4d0a86be7494cc3103eb_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fe20c1cf817a4d0a86be7494cc3103eb_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:42,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/adb21bbf9e354cd7a727e24080b9dcfd, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/adb21bbf9e354cd7a727e24080b9dcfd is 175, key is test_row_0/A:col10/1732664080508/Put/seqid=0 2024-11-26T23:34:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742150_1326 (size=30955) 2024-11-26T23:34:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T23:34:42,655 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/adb21bbf9e354cd7a727e24080b9dcfd 2024-11-26T23:34:42,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/76055f727e934aa5bad59263b7be43f7 is 50, key is test_row_0/B:col10/1732664080508/Put/seqid=0 2024-11-26T23:34:42,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742151_1327 (size=12001) 2024-11-26T23:34:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:42,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:42,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664142708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664142709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664142711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664142711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664142712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664142821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664142831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664142831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664142831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664142831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664143041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664143051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664143052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664143052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664143052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,068 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/76055f727e934aa5bad59263b7be43f7 2024-11-26T23:34:43,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/aa621c1a28954e7483bc45cfdc23b4fb is 50, key is test_row_0/C:col10/1732664080508/Put/seqid=0 2024-11-26T23:34:43,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742152_1328 (size=12001) 2024-11-26T23:34:43,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664143362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664143362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664143362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664143362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664143371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/aa621c1a28954e7483bc45cfdc23b4fb 2024-11-26T23:34:43,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/adb21bbf9e354cd7a727e24080b9dcfd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/adb21bbf9e354cd7a727e24080b9dcfd 2024-11-26T23:34:43,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/adb21bbf9e354cd7a727e24080b9dcfd, entries=150, sequenceid=41, filesize=30.2 K 2024-11-26T23:34:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/76055f727e934aa5bad59263b7be43f7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/76055f727e934aa5bad59263b7be43f7 2024-11-26T23:34:43,491 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/76055f727e934aa5bad59263b7be43f7, entries=150, sequenceid=41, filesize=11.7 K 2024-11-26T23:34:43,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/aa621c1a28954e7483bc45cfdc23b4fb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/aa621c1a28954e7483bc45cfdc23b4fb 2024-11-26T23:34:43,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/aa621c1a28954e7483bc45cfdc23b4fb, entries=150, sequenceid=41, filesize=11.7 K 2024-11-26T23:34:43,496 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1670ms, sequenceid=41, compaction requested=false 2024-11-26T23:34:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-26T23:34:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-26T23:34:43,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-26T23:34:43,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0520 sec 2024-11-26T23:34:43,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.0610 sec 2024-11-26T23:34:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:43,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:43,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-26T23:34:43,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112636c6b5c717b94d5cacf53143b3df2a86_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664083886/Put/seqid=0 2024-11-26T23:34:43,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742153_1329 (size=14594) 2024-11-26T23:34:43,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664143923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664143924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664143925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664143926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664143929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664144042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664144042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664144042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664144042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664144052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664144261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664144261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664144261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664144261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664144271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,300 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:44,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112636c6b5c717b94d5cacf53143b3df2a86_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112636c6b5c717b94d5cacf53143b3df2a86_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:44,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/717da68741014308ad3d28575cb1e099, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:44,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/717da68741014308ad3d28575cb1e099 is 175, key is test_row_0/A:col10/1732664083886/Put/seqid=0 2024-11-26T23:34:44,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742154_1330 (size=39549) 2024-11-26T23:34:44,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/717da68741014308ad3d28575cb1e099 2024-11-26T23:34:44,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/b37c3f75b9fc4e11a85c8e02baa92358 is 50, key is test_row_0/B:col10/1732664083886/Put/seqid=0 2024-11-26T23:34:44,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742155_1331 (size=12001) 2024-11-26T23:34:44,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664144572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664144572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664144581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664144581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664144591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T23:34:44,612 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:44,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-26T23:34:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T23:34:44,614 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:44,615 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:44,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:44,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/b37c3f75b9fc4e11a85c8e02baa92358 2024-11-26T23:34:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T23:34:44,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/5b17f9e130f940dca239b4ad6b799cf5 is 50, key is test_row_0/C:col10/1732664083886/Put/seqid=0 2024-11-26T23:34:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742156_1332 (size=12001) 2024-11-26T23:34:44,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T23:34:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:44,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:44,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T23:34:44,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:44,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:44,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:44,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:44,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T23:34:45,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T23:34:45,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:45,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:45,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:45,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:45,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:45,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664145092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:45,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664145092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664145092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664145092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664145112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:45,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/5b17f9e130f940dca239b4ad6b799cf5 2024-11-26T23:34:45,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/717da68741014308ad3d28575cb1e099 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/717da68741014308ad3d28575cb1e099 2024-11-26T23:34:45,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/717da68741014308ad3d28575cb1e099, entries=200, sequenceid=56, filesize=38.6 K 2024-11-26T23:34:45,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/b37c3f75b9fc4e11a85c8e02baa92358 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/b37c3f75b9fc4e11a85c8e02baa92358 2024-11-26T23:34:45,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/b37c3f75b9fc4e11a85c8e02baa92358, entries=150, sequenceid=56, filesize=11.7 K 2024-11-26T23:34:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/5b17f9e130f940dca239b4ad6b799cf5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5b17f9e130f940dca239b4ad6b799cf5 2024-11-26T23:34:45,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5b17f9e130f940dca239b4ad6b799cf5, entries=150, sequenceid=56, filesize=11.7 K 2024-11-26T23:34:45,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1261ms, sequenceid=56, compaction requested=true 2024-11-26T23:34:45,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:45,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:45,149 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:45,149 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:45,149 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:45,149 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:45,150 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/B is initiating minor compaction (all files) 2024-11-26T23:34:45,150 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/A is initiating minor compaction (all files) 2024-11-26T23:34:45,150 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/B in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:45,150 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/A in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:45,150 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/0fc1251dbc1f4045bf03daaf440001c6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/76055f727e934aa5bad59263b7be43f7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/b37c3f75b9fc4e11a85c8e02baa92358] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=35.2 K 2024-11-26T23:34:45,150 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/704ccf06f89a4b76ac2c4fca4f572b71, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/adb21bbf9e354cd7a727e24080b9dcfd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/717da68741014308ad3d28575cb1e099] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=107.5 K 2024-11-26T23:34:45,150 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:45,150 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/704ccf06f89a4b76ac2c4fca4f572b71, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/adb21bbf9e354cd7a727e24080b9dcfd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/717da68741014308ad3d28575cb1e099] 2024-11-26T23:34:45,150 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc1251dbc1f4045bf03daaf440001c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732664080460 2024-11-26T23:34:45,150 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 704ccf06f89a4b76ac2c4fca4f572b71, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732664080448 2024-11-26T23:34:45,150 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 76055f727e934aa5bad59263b7be43f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732664080492 2024-11-26T23:34:45,151 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb21bbf9e354cd7a727e24080b9dcfd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732664080492 2024-11-26T23:34:45,151 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b37c3f75b9fc4e11a85c8e02baa92358, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732664082709 2024-11-26T23:34:45,151 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717da68741014308ad3d28575cb1e099, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732664082709 2024-11-26T23:34:45,156 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:45,157 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#B#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:45,158 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/f3af40cb010c4b6c854caea27bfdb08f is 50, key is test_row_0/B:col10/1732664083886/Put/seqid=0 2024-11-26T23:34:45,160 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126e91fe151350c466dae732fdb7afab3f2_38491e1fe242cb24ed1e0e85404fb0f7 store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:45,161 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126e91fe151350c466dae732fdb7afab3f2_38491e1fe242cb24ed1e0e85404fb0f7, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:45,162 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e91fe151350c466dae732fdb7afab3f2_38491e1fe242cb24ed1e0e85404fb0f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:45,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742157_1333 (size=12104) 2024-11-26T23:34:45,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742158_1334 (size=4469) 2024-11-26T23:34:45,176 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#A#compaction#288 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:45,176 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/ccad19a4d5784d04af951467a8a12915 is 175, key is test_row_0/A:col10/1732664083886/Put/seqid=0 2024-11-26T23:34:45,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742159_1335 (size=31058) 2024-11-26T23:34:45,184 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/ccad19a4d5784d04af951467a8a12915 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ccad19a4d5784d04af951467a8a12915 2024-11-26T23:34:45,190 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/A of 38491e1fe242cb24ed1e0e85404fb0f7 into ccad19a4d5784d04af951467a8a12915(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:45,190 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:45,190 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/A, priority=13, startTime=1732664085148; duration=0sec 2024-11-26T23:34:45,190 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:45,190 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:A 2024-11-26T23:34:45,190 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:45,191 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:45,191 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/C is initiating minor compaction (all files) 2024-11-26T23:34:45,191 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/C in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:45,191 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2854896927a74dc49775230c121e9090, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/aa621c1a28954e7483bc45cfdc23b4fb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5b17f9e130f940dca239b4ad6b799cf5] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=35.2 K 2024-11-26T23:34:45,191 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2854896927a74dc49775230c121e9090, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732664080460 2024-11-26T23:34:45,192 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa621c1a28954e7483bc45cfdc23b4fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732664080492 2024-11-26T23:34:45,192 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b17f9e130f940dca239b4ad6b799cf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732664082709 2024-11-26T23:34:45,199 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#C#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:45,199 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/56770bdca20f4f55982c60ff4764cedf is 50, key is test_row_0/C:col10/1732664083886/Put/seqid=0 2024-11-26T23:34:45,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742160_1336 (size=12104) 2024-11-26T23:34:45,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T23:34:45,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:45,227 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-26T23:34:45,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265af36ede07b24b60bbbb32919da48098_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664083925/Put/seqid=0 2024-11-26T23:34:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742161_1337 (size=12154) 2024-11-26T23:34:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T23:34:45,570 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/f3af40cb010c4b6c854caea27bfdb08f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f3af40cb010c4b6c854caea27bfdb08f 2024-11-26T23:34:45,574 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/B of 38491e1fe242cb24ed1e0e85404fb0f7 into f3af40cb010c4b6c854caea27bfdb08f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:45,574 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:45,574 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/B, priority=13, startTime=1732664085149; duration=0sec 2024-11-26T23:34:45,575 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:45,575 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:B 2024-11-26T23:34:45,613 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/56770bdca20f4f55982c60ff4764cedf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/56770bdca20f4f55982c60ff4764cedf 2024-11-26T23:34:45,617 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/C of 38491e1fe242cb24ed1e0e85404fb0f7 into 56770bdca20f4f55982c60ff4764cedf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:45,617 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:45,617 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/C, priority=13, startTime=1732664085149; duration=0sec 2024-11-26T23:34:45,617 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:45,617 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:C 2024-11-26T23:34:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:45,644 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265af36ede07b24b60bbbb32919da48098_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265af36ede07b24b60bbbb32919da48098_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/0c8a097f09e0414b88296265ec4f1778, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/0c8a097f09e0414b88296265ec4f1778 is 175, key is test_row_0/A:col10/1732664083925/Put/seqid=0 2024-11-26T23:34:45,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742162_1338 (size=30955) 2024-11-26T23:34:45,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/0c8a097f09e0414b88296265ec4f1778 2024-11-26T23:34:45,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/7c7be85a189e492f8257eb7965bf5aab is 50, key is test_row_0/B:col10/1732664083925/Put/seqid=0 2024-11-26T23:34:45,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742163_1339 (size=12001) 2024-11-26T23:34:45,662 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/7c7be85a189e492f8257eb7965bf5aab 2024-11-26T23:34:45,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/04d7634b8068419facbfbfd374fadac0 is 50, key is test_row_0/C:col10/1732664083925/Put/seqid=0 2024-11-26T23:34:45,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742164_1340 (size=12001) 2024-11-26T23:34:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T23:34:45,891 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:34:46,073 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/04d7634b8068419facbfbfd374fadac0 2024-11-26T23:34:46,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/0c8a097f09e0414b88296265ec4f1778 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0c8a097f09e0414b88296265ec4f1778 2024-11-26T23:34:46,081 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0c8a097f09e0414b88296265ec4f1778, entries=150, sequenceid=78, filesize=30.2 K 2024-11-26T23:34:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/7c7be85a189e492f8257eb7965bf5aab as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/7c7be85a189e492f8257eb7965bf5aab 2024-11-26T23:34:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/7c7be85a189e492f8257eb7965bf5aab, entries=150, sequenceid=78, filesize=11.7 K 2024-11-26T23:34:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/04d7634b8068419facbfbfd374fadac0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/04d7634b8068419facbfbfd374fadac0 2024-11-26T23:34:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/04d7634b8068419facbfbfd374fadac0, entries=150, sequenceid=78, filesize=11.7 K 2024-11-26T23:34:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 38491e1fe242cb24ed1e0e85404fb0f7 in 872ms, sequenceid=78, compaction requested=false 2024-11-26T23:34:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-26T23:34:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-26T23:34:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-26T23:34:46,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-11-26T23:34:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.4910 sec 2024-11-26T23:34:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T23:34:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,144 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T23:34:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,222 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d9c44c64ef52431e975029521f3f40f9_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664086220/Put/seqid=0 2024-11-26T23:34:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742165_1341 (size=21918) 2024-11-26T23:34:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,237 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d9c44c64ef52431e975029521f3f40f9_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d9c44c64ef52431e975029521f3f40f9_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/52f8d242941d4a0c96e2e1854c9f3e56, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/52f8d242941d4a0c96e2e1854c9f3e56 is 175, key is test_row_0/A:col10/1732664086220/Put/seqid=0 2024-11-26T23:34:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742166_1342 (size=65323) 2024-11-26T23:34:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,250 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/52f8d242941d4a0c96e2e1854c9f3e56 2024-11-26T23:34:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:46,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/bd44bc3ba08e4d4289a7a1001f57ad9a is 50, key is test_row_0/B:col10/1732664086220/Put/seqid=0 2024-11-26T23:34:46,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742167_1343 (size=12001) 2024-11-26T23:34:46,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664146282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664146291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664146291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664146294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664146295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664146402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664146411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664146411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664146411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664146411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664146611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664146621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664146621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664146621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664146632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/bd44bc3ba08e4d4289a7a1001f57ad9a 2024-11-26T23:34:46,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/7b157572ac704714ae4eafabf5141cd3 is 50, key is test_row_0/C:col10/1732664086220/Put/seqid=0 2024-11-26T23:34:46,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742168_1344 (size=12001) 2024-11-26T23:34:46,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/7b157572ac704714ae4eafabf5141cd3 2024-11-26T23:34:46,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/52f8d242941d4a0c96e2e1854c9f3e56 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/52f8d242941d4a0c96e2e1854c9f3e56 2024-11-26T23:34:46,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/52f8d242941d4a0c96e2e1854c9f3e56, entries=350, sequenceid=92, filesize=63.8 K 2024-11-26T23:34:46,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/bd44bc3ba08e4d4289a7a1001f57ad9a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/bd44bc3ba08e4d4289a7a1001f57ad9a 2024-11-26T23:34:46,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/bd44bc3ba08e4d4289a7a1001f57ad9a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-26T23:34:46,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/7b157572ac704714ae4eafabf5141cd3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/7b157572ac704714ae4eafabf5141cd3 2024-11-26T23:34:46,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/7b157572ac704714ae4eafabf5141cd3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-26T23:34:46,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 38491e1fe242cb24ed1e0e85404fb0f7 in 478ms, sequenceid=92, compaction requested=true 2024-11-26T23:34:46,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:46,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:46,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:46,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:46,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:46,700 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:46,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:46,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:46,700 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:46,701 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:46,702 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/A is initiating minor compaction (all files) 2024-11-26T23:34:46,702 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/A in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:46,702 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ccad19a4d5784d04af951467a8a12915, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0c8a097f09e0414b88296265ec4f1778, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/52f8d242941d4a0c96e2e1854c9f3e56] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=124.4 K 2024-11-26T23:34:46,702 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:46,702 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ccad19a4d5784d04af951467a8a12915, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0c8a097f09e0414b88296265ec4f1778, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/52f8d242941d4a0c96e2e1854c9f3e56] 2024-11-26T23:34:46,702 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:46,702 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/B is initiating minor compaction (all files) 2024-11-26T23:34:46,702 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/B in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:46,702 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f3af40cb010c4b6c854caea27bfdb08f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/7c7be85a189e492f8257eb7965bf5aab, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/bd44bc3ba08e4d4289a7a1001f57ad9a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=35.3 K 2024-11-26T23:34:46,702 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccad19a4d5784d04af951467a8a12915, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732664082709 2024-11-26T23:34:46,702 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f3af40cb010c4b6c854caea27bfdb08f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732664082709 2024-11-26T23:34:46,703 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8a097f09e0414b88296265ec4f1778, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732664083924 2024-11-26T23:34:46,703 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7be85a189e492f8257eb7965bf5aab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732664083924 2024-11-26T23:34:46,703 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f8d242941d4a0c96e2e1854c9f3e56, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732664086168 2024-11-26T23:34:46,703 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting bd44bc3ba08e4d4289a7a1001f57ad9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732664086217 2024-11-26T23:34:46,716 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:46,717 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#B#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:46,718 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/4e9710b45ace43b4bf5d793e94635fda is 50, key is test_row_0/B:col10/1732664086220/Put/seqid=0 2024-11-26T23:34:46,718 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126c1f97bd292d54543b0a7a200ad6c0c9a_38491e1fe242cb24ed1e0e85404fb0f7 store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:46,721 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126c1f97bd292d54543b0a7a200ad6c0c9a_38491e1fe242cb24ed1e0e85404fb0f7, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:46,721 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c1f97bd292d54543b0a7a200ad6c0c9a_38491e1fe242cb24ed1e0e85404fb0f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:46,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742169_1345 (size=12207) 2024-11-26T23:34:46,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742170_1346 (size=4469) 2024-11-26T23:34:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T23:34:46,762 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:46,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-26T23:34:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T23:34:46,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:46,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:46,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T23:34:46,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T23:34:46,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:46,918 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:34:46,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269e75a8d4a21f4572bae38727b294df82_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664086294/Put/seqid=0 2024-11-26T23:34:46,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742171_1347 (size=12154) 2024-11-26T23:34:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:46,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664146957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664146958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664146958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664146969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:46,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664146970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:46,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:46,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T23:34:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664147082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664147082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664147082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664147082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664147092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,130 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/4e9710b45ace43b4bf5d793e94635fda as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/4e9710b45ace43b4bf5d793e94635fda 2024-11-26T23:34:47,135 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#A#compaction#297 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:47,135 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/814be5d1fd2d40a096e8032c86ce5f01 is 175, key is test_row_0/A:col10/1732664086220/Put/seqid=0 2024-11-26T23:34:47,139 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/B of 38491e1fe242cb24ed1e0e85404fb0f7 into 4e9710b45ace43b4bf5d793e94635fda(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:47,139 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:47,139 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/B, priority=13, startTime=1732664086700; duration=0sec 2024-11-26T23:34:47,139 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:47,139 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:B 2024-11-26T23:34:47,139 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:47,140 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:47,140 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/C is initiating minor compaction (all files) 2024-11-26T23:34:47,140 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/C in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:47,140 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/56770bdca20f4f55982c60ff4764cedf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/04d7634b8068419facbfbfd374fadac0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/7b157572ac704714ae4eafabf5141cd3] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=35.3 K 2024-11-26T23:34:47,140 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 56770bdca20f4f55982c60ff4764cedf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732664082709 2024-11-26T23:34:47,141 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d7634b8068419facbfbfd374fadac0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732664083924 2024-11-26T23:34:47,141 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b157572ac704714ae4eafabf5141cd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732664086217 2024-11-26T23:34:47,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742172_1348 (size=31161) 2024-11-26T23:34:47,155 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/814be5d1fd2d40a096e8032c86ce5f01 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/814be5d1fd2d40a096e8032c86ce5f01 2024-11-26T23:34:47,158 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:47,160 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/afd467c593b843cf8dd995a130b71059 is 50, key is test_row_0/C:col10/1732664086220/Put/seqid=0 2024-11-26T23:34:47,165 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/A of 38491e1fe242cb24ed1e0e85404fb0f7 into 814be5d1fd2d40a096e8032c86ce5f01(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:47,165 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:47,165 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/A, priority=13, startTime=1732664086700; duration=0sec 2024-11-26T23:34:47,166 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:47,166 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:A 2024-11-26T23:34:47,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742173_1349 (size=12207) 2024-11-26T23:34:47,176 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/afd467c593b843cf8dd995a130b71059 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/afd467c593b843cf8dd995a130b71059 2024-11-26T23:34:47,182 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/C of 38491e1fe242cb24ed1e0e85404fb0f7 into afd467c593b843cf8dd995a130b71059(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:47,182 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:47,182 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/C, priority=13, startTime=1732664086700; duration=0sec 2024-11-26T23:34:47,182 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:47,182 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:C 2024-11-26T23:34:47,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664147302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664147302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664147302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664147302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664147302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:47,332 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269e75a8d4a21f4572bae38727b294df82_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269e75a8d4a21f4572bae38727b294df82_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/203a5ab149d9417d9e4d7ab2df8d4194, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/203a5ab149d9417d9e4d7ab2df8d4194 is 175, key is test_row_0/A:col10/1732664086294/Put/seqid=0 2024-11-26T23:34:47,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742174_1350 (size=30955) 2024-11-26T23:34:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T23:34:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664147623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664147623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664147623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664147624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664147624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:47,737 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/203a5ab149d9417d9e4d7ab2df8d4194 2024-11-26T23:34:47,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/5a88932207ae49a2a9839f9bbf6771bd is 50, key is test_row_0/B:col10/1732664086294/Put/seqid=0 2024-11-26T23:34:47,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742175_1351 (size=12001) 2024-11-26T23:34:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T23:34:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664148131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:48,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664148142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:48,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664148142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:48,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664148142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:48,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664148142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:48,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/5a88932207ae49a2a9839f9bbf6771bd 2024-11-26T23:34:48,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/3c7355607fbb45afaa23f5866c8b9951 is 50, key is test_row_0/C:col10/1732664086294/Put/seqid=0 2024-11-26T23:34:48,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742176_1352 (size=12001) 2024-11-26T23:34:48,563 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/3c7355607fbb45afaa23f5866c8b9951 2024-11-26T23:34:48,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/203a5ab149d9417d9e4d7ab2df8d4194 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/203a5ab149d9417d9e4d7ab2df8d4194 2024-11-26T23:34:48,571 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/203a5ab149d9417d9e4d7ab2df8d4194, entries=150, sequenceid=116, filesize=30.2 K 2024-11-26T23:34:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/5a88932207ae49a2a9839f9bbf6771bd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5a88932207ae49a2a9839f9bbf6771bd 2024-11-26T23:34:48,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5a88932207ae49a2a9839f9bbf6771bd, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T23:34:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/3c7355607fbb45afaa23f5866c8b9951 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3c7355607fbb45afaa23f5866c8b9951 2024-11-26T23:34:48,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3c7355607fbb45afaa23f5866c8b9951, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T23:34:48,590 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1671ms, sequenceid=116, compaction requested=false 2024-11-26T23:34:48,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:48,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:48,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-26T23:34:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-26T23:34:48,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-26T23:34:48,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8260 sec 2024-11-26T23:34:48,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8320 sec 2024-11-26T23:34:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T23:34:48,912 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:48,912 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-26T23:34:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T23:34:48,914 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:48,915 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:48,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T23:34:49,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T23:34:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:49,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T23:34:49,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266511f0010bda4864a5b4c40b77584b5a_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664086969/Put/seqid=0 2024-11-26T23:34:49,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742177_1353 (size=12154) 2024-11-26T23:34:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:49,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T23:34:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664149233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664149245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664149246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664149247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664149248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664149351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664149362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664149362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664149362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664149371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:49,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266511f0010bda4864a5b4c40b77584b5a_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266511f0010bda4864a5b4c40b77584b5a_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2beac782bf8441ea9e801151a3062dd6, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2beac782bf8441ea9e801151a3062dd6 is 175, key is test_row_0/A:col10/1732664086969/Put/seqid=0 2024-11-26T23:34:49,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742178_1354 (size=30955) 2024-11-26T23:34:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T23:34:49,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664149562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664149571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664149581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664149581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664149592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,888 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2beac782bf8441ea9e801151a3062dd6 2024-11-26T23:34:49,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664149882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/64ed3e8c414d4b259e02f1d306c0b6e5 is 50, key is test_row_0/B:col10/1732664086969/Put/seqid=0 2024-11-26T23:34:49,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742179_1355 (size=12001) 2024-11-26T23:34:49,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664149891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664149901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664149901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:49,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664149912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T23:34:50,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/64ed3e8c414d4b259e02f1d306c0b6e5 2024-11-26T23:34:50,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/d6168be381bf4f64939efd38ca1f396a is 50, key is test_row_0/C:col10/1732664086969/Put/seqid=0 2024-11-26T23:34:50,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742180_1356 (size=12001) 2024-11-26T23:34:50,311 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/d6168be381bf4f64939efd38ca1f396a 2024-11-26T23:34:50,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2beac782bf8441ea9e801151a3062dd6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2beac782bf8441ea9e801151a3062dd6 2024-11-26T23:34:50,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2beac782bf8441ea9e801151a3062dd6, entries=150, sequenceid=131, filesize=30.2 K 2024-11-26T23:34:50,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/64ed3e8c414d4b259e02f1d306c0b6e5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/64ed3e8c414d4b259e02f1d306c0b6e5 2024-11-26T23:34:50,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/64ed3e8c414d4b259e02f1d306c0b6e5, entries=150, sequenceid=131, filesize=11.7 K 2024-11-26T23:34:50,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/d6168be381bf4f64939efd38ca1f396a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/d6168be381bf4f64939efd38ca1f396a 2024-11-26T23:34:50,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/d6168be381bf4f64939efd38ca1f396a, entries=150, sequenceid=131, filesize=11.7 K 2024-11-26T23:34:50,330 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1263ms, sequenceid=131, compaction requested=true 2024-11-26T23:34:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-26T23:34:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-26T23:34:50,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-26T23:34:50,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-26T23:34:50,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4220 sec 2024-11-26T23:34:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:50,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:50,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T23:34:50,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126db721c53744d4c7180fefccbc4e40199_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664090406/Put/seqid=0 2024-11-26T23:34:50,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742181_1357 (size=14794) 2024-11-26T23:34:50,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664150421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664150422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664150431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664150431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664150432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664150541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664150541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664150541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664150552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664150751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664150751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664150751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664150762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:50,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:50,823 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126db721c53744d4c7180fefccbc4e40199_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126db721c53744d4c7180fefccbc4e40199_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:50,824 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/a3b79807f39147d9af0497ec8959ab32, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:50,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/a3b79807f39147d9af0497ec8959ab32 is 175, key is test_row_0/A:col10/1732664090406/Put/seqid=0 2024-11-26T23:34:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742182_1358 (size=39749) 2024-11-26T23:34:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T23:34:51,061 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:51,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-26T23:34:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T23:34:51,064 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:51,064 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:51,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:51,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664151061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664151061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664151071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664151082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T23:34:51,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T23:34:51,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:51,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/a3b79807f39147d9af0497ec8959ab32 2024-11-26T23:34:51,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/8858a32803b3498a9c4d1eba6fd1a97a is 50, key is test_row_0/B:col10/1732664090406/Put/seqid=0 2024-11-26T23:34:51,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742183_1359 (size=12151) 2024-11-26T23:34:51,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T23:34:51,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T23:34:51,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664151461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,470 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 38491e1fe242cb24ed1e0e85404fb0f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:51,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T23:34:51,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:51,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664151571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664151571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664151592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664151602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:51,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/8858a32803b3498a9c4d1eba6fd1a97a 2024-11-26T23:34:51,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/89ec5328434d4a988b19cd566edead59 is 50, key is test_row_0/C:col10/1732664090406/Put/seqid=0 2024-11-26T23:34:51,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742184_1360 (size=12151) 2024-11-26T23:34:51,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T23:34:51,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:51,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T23:34:51,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T23:34:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T23:34:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:51,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:52,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/89ec5328434d4a988b19cd566edead59 2024-11-26T23:34:52,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/a3b79807f39147d9af0497ec8959ab32 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a3b79807f39147d9af0497ec8959ab32 2024-11-26T23:34:52,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a3b79807f39147d9af0497ec8959ab32, entries=200, sequenceid=157, filesize=38.8 K 2024-11-26T23:34:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/8858a32803b3498a9c4d1eba6fd1a97a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/8858a32803b3498a9c4d1eba6fd1a97a 2024-11-26T23:34:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/8858a32803b3498a9c4d1eba6fd1a97a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T23:34:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/89ec5328434d4a988b19cd566edead59 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/89ec5328434d4a988b19cd566edead59 2024-11-26T23:34:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/89ec5328434d4a988b19cd566edead59, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T23:34:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1665ms, sequenceid=157, compaction requested=true 2024-11-26T23:34:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:52,072 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:52,072 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,074 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:52,074 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/A is initiating minor compaction (all files) 2024-11-26T23:34:52,074 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/A in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:52,074 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/814be5d1fd2d40a096e8032c86ce5f01, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/203a5ab149d9417d9e4d7ab2df8d4194, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2beac782bf8441ea9e801151a3062dd6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a3b79807f39147d9af0497ec8959ab32] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=129.7 K 2024-11-26T23:34:52,074 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:52,074 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/814be5d1fd2d40a096e8032c86ce5f01, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/203a5ab149d9417d9e4d7ab2df8d4194, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2beac782bf8441ea9e801151a3062dd6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a3b79807f39147d9af0497ec8959ab32] 2024-11-26T23:34:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,074 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 814be5d1fd2d40a096e8032c86ce5f01, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732664086217 2024-11-26T23:34:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,074 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:52,074 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/B is initiating minor compaction (all files) 2024-11-26T23:34:52,075 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/B in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:52,075 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/4e9710b45ace43b4bf5d793e94635fda, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5a88932207ae49a2a9839f9bbf6771bd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/64ed3e8c414d4b259e02f1d306c0b6e5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/8858a32803b3498a9c4d1eba6fd1a97a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=47.2 K 2024-11-26T23:34:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,075 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203a5ab149d9417d9e4d7ab2df8d4194, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732664086276 2024-11-26T23:34:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,075 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9710b45ace43b4bf5d793e94635fda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732664086217 2024-11-26T23:34:52,075 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2beac782bf8441ea9e801151a3062dd6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732664086939 2024-11-26T23:34:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,075 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b79807f39147d9af0497ec8959ab32, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732664089232 2024-11-26T23:34:52,075 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a88932207ae49a2a9839f9bbf6771bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732664086276 2024-11-26T23:34:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,076 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ed3e8c414d4b259e02f1d306c0b6e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732664086939 2024-11-26T23:34:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,076 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8858a32803b3498a9c4d1eba6fd1a97a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732664089246 2024-11-26T23:34:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,082 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,091 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126ed8eda1250ed4c7a85354f152cd42933_38491e1fe242cb24ed1e0e85404fb0f7 store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,093 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126ed8eda1250ed4c7a85354f152cd42933_38491e1fe242cb24ed1e0e85404fb0f7, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,093 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:52,093 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ed8eda1250ed4c7a85354f152cd42933_38491e1fe242cb24ed1e0e85404fb0f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,093 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/67d349a0653e460697a98e58ea122297 is 50, key is test_row_0/B:col10/1732664090406/Put/seqid=0 2024-11-26T23:34:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742185_1361 (size=4469) 2024-11-26T23:34:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742186_1362 (size=12493) 2024-11-26T23:34:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T23:34:52,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T23:34:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112637009f437e4b46508bd16dbea0d9f89a_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664090431/Put/seqid=0 2024-11-26T23:34:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742187_1363 (size=9814) 2024-11-26T23:34:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T23:34:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,505 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#A#compaction#308 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,506 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/9fd3cf79a1214526bea98828d89f6222 is 175, key is test_row_0/A:col10/1732664090406/Put/seqid=0 2024-11-26T23:34:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742188_1364 (size=31447) 2024-11-26T23:34:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,518 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/67d349a0653e460697a98e58ea122297 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/67d349a0653e460697a98e58ea122297 2024-11-26T23:34:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,524 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/B of 38491e1fe242cb24ed1e0e85404fb0f7 into 67d349a0653e460697a98e58ea122297(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:52,524 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:52,524 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/B, priority=12, startTime=1732664092072; duration=0sec 2024-11-26T23:34:52,524 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:52,524 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:B 2024-11-26T23:34:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,524 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,525 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:52,526 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/C is initiating minor compaction (all files) 2024-11-26T23:34:52,526 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/C in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:52,526 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/afd467c593b843cf8dd995a130b71059, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3c7355607fbb45afaa23f5866c8b9951, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/d6168be381bf4f64939efd38ca1f396a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/89ec5328434d4a988b19cd566edead59] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=47.2 K 2024-11-26T23:34:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,526 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting afd467c593b843cf8dd995a130b71059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732664086217 2024-11-26T23:34:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,526 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c7355607fbb45afaa23f5866c8b9951, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732664086276 2024-11-26T23:34:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,527 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d6168be381bf4f64939efd38ca1f396a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732664086939 2024-11-26T23:34:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,527 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 89ec5328434d4a988b19cd566edead59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732664089246 2024-11-26T23:34:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,538 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,539 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/fe815faa0eeb48bba820a06012d74e8f is 50, key is test_row_0/C:col10/1732664090406/Put/seqid=0 2024-11-26T23:34:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742189_1365 (size=12493) 2024-11-26T23:34:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,555 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112637009f437e4b46508bd16dbea0d9f89a_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112637009f437e4b46508bd16dbea0d9f89a_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6e18772f7cd34240bfc7364f413780e0, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6e18772f7cd34240bfc7364f413780e0 is 175, key is test_row_0/A:col10/1732664090431/Put/seqid=0 2024-11-26T23:34:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,559 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/fe815faa0eeb48bba820a06012d74e8f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/fe815faa0eeb48bba820a06012d74e8f 2024-11-26T23:34:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,566 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/C of 38491e1fe242cb24ed1e0e85404fb0f7 into fe815faa0eeb48bba820a06012d74e8f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,566 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:52,566 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/C, priority=12, startTime=1732664092072; duration=0sec 2024-11-26T23:34:52,566 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:52,566 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:C 2024-11-26T23:34:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742190_1366 (size=22461) 2024-11-26T23:34:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,572 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6e18772f7cd34240bfc7364f413780e0 2024-11-26T23:34:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/98369526f34f4817899e9c0f5e154f87 is 50, key is test_row_0/B:col10/1732664090431/Put/seqid=0 2024-11-26T23:34:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742191_1367 (size=9757) 2024-11-26T23:34:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:52,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664152692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664152702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664152707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664152708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664152811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664152811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664152822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664152822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:52,916 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/9fd3cf79a1214526bea98828d89f6222 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/9fd3cf79a1214526bea98828d89f6222 2024-11-26T23:34:52,922 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/A of 38491e1fe242cb24ed1e0e85404fb0f7 into 9fd3cf79a1214526bea98828d89f6222(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:52,922 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:52,922 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/A, priority=12, startTime=1732664092072; duration=0sec 2024-11-26T23:34:52,922 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:52,922 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:A 2024-11-26T23:34:52,999 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/98369526f34f4817899e9c0f5e154f87 2024-11-26T23:34:53,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/05c3dbbf33634156a1de193bce3d8e08 is 50, key is test_row_0/C:col10/1732664090431/Put/seqid=0 2024-11-26T23:34:53,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742192_1368 (size=9757) 2024-11-26T23:34:53,011 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/05c3dbbf33634156a1de193bce3d8e08 2024-11-26T23:34:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6e18772f7cd34240bfc7364f413780e0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6e18772f7cd34240bfc7364f413780e0 2024-11-26T23:34:53,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6e18772f7cd34240bfc7364f413780e0, entries=100, sequenceid=167, filesize=21.9 K 2024-11-26T23:34:53,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/98369526f34f4817899e9c0f5e154f87 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/98369526f34f4817899e9c0f5e154f87 2024-11-26T23:34:53,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/98369526f34f4817899e9c0f5e154f87, entries=100, sequenceid=167, filesize=9.5 K 2024-11-26T23:34:53,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/05c3dbbf33634156a1de193bce3d8e08 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/05c3dbbf33634156a1de193bce3d8e08 2024-11-26T23:34:53,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664153021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664153021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,033 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/05c3dbbf33634156a1de193bce3d8e08, entries=100, sequenceid=167, filesize=9.5 K 2024-11-26T23:34:53,034 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 38491e1fe242cb24ed1e0e85404fb0f7 in 899ms, sequenceid=167, compaction requested=false 2024-11-26T23:34:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-26T23:34:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-26T23:34:53,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-26T23:34:53,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9710 sec 2024-11-26T23:34:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:53,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9760 sec 2024-11-26T23:34:53,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-26T23:34:53,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265e306deb67b64982b220865a49332ab8_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664093039/Put/seqid=0 2024-11-26T23:34:53,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742193_1369 (size=14794) 2024-11-26T23:34:53,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:53,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265e306deb67b64982b220865a49332ab8_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265e306deb67b64982b220865a49332ab8_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:53,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/fa59b2d621ae4d159ae952b134772c22, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:53,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/fa59b2d621ae4d159ae952b134772c22 is 175, key is test_row_0/A:col10/1732664093039/Put/seqid=0 2024-11-26T23:34:53,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664153066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664153067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742194_1370 (size=39749) 2024-11-26T23:34:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664153181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664153181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T23:34:53,212 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:53,212 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-26T23:34:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T23:34:53,214 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:53,215 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:53,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T23:34:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664153342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664153342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664153391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664153402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/fa59b2d621ae4d159ae952b134772c22 2024-11-26T23:34:53,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/ad90ad7da13c4937872124748eaed24e is 50, key is test_row_0/B:col10/1732664093039/Put/seqid=0 2024-11-26T23:34:53,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742195_1371 (size=12151) 2024-11-26T23:34:53,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664153491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,498 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 38491e1fe242cb24ed1e0e85404fb0f7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:53,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:53,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T23:34:53,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:53,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:53,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664153701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664153711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:53,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:53,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T23:34:53,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664153862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664153862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:53,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/ad90ad7da13c4937872124748eaed24e 2024-11-26T23:34:53,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/3dcdce0058d54df5947623c5ffc6e8af is 50, key is test_row_0/C:col10/1732664093039/Put/seqid=0 2024-11-26T23:34:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742196_1372 (size=12151) 2024-11-26T23:34:53,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:53,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:53,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:53,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:54,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:54,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664154211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:54,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664154222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:54,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:54,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:54,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:54,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:54,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/3dcdce0058d54df5947623c5ffc6e8af 2024-11-26T23:34:54,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/fa59b2d621ae4d159ae952b134772c22 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/fa59b2d621ae4d159ae952b134772c22 2024-11-26T23:34:54,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/fa59b2d621ae4d159ae952b134772c22, entries=200, sequenceid=199, filesize=38.8 K 2024-11-26T23:34:54,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/ad90ad7da13c4937872124748eaed24e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/ad90ad7da13c4937872124748eaed24e 2024-11-26T23:34:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/ad90ad7da13c4937872124748eaed24e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-26T23:34:54,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/3dcdce0058d54df5947623c5ffc6e8af as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3dcdce0058d54df5947623c5ffc6e8af 2024-11-26T23:34:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3dcdce0058d54df5947623c5ffc6e8af, entries=150, sequenceid=199, filesize=11.9 K 2024-11-26T23:34:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1279ms, sequenceid=199, compaction requested=true 2024-11-26T23:34:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:54,318 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:54,318 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,320 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:54,320 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/B is initiating minor compaction (all files) 2024-11-26T23:34:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,320 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:54,320 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/B in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,321 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/A is initiating minor compaction (all files) 2024-11-26T23:34:54,321 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/67d349a0653e460697a98e58ea122297, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/98369526f34f4817899e9c0f5e154f87, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/ad90ad7da13c4937872124748eaed24e] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=33.6 K 2024-11-26T23:34:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,321 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/A in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,321 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/9fd3cf79a1214526bea98828d89f6222, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6e18772f7cd34240bfc7364f413780e0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/fa59b2d621ae4d159ae952b134772c22] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=91.5 K 2024-11-26T23:34:54,321 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,321 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/9fd3cf79a1214526bea98828d89f6222, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6e18772f7cd34240bfc7364f413780e0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/fa59b2d621ae4d159ae952b134772c22] 2024-11-26T23:34:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,321 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d349a0653e460697a98e58ea122297, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732664089246 2024-11-26T23:34:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,321 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd3cf79a1214526bea98828d89f6222, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732664089246 2024-11-26T23:34:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,322 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e18772f7cd34240bfc7364f413780e0, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732664090430 2024-11-26T23:34:54,322 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 98369526f34f4817899e9c0f5e154f87, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732664090430 2024-11-26T23:34:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,322 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa59b2d621ae4d159ae952b134772c22, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664092662 2024-11-26T23:34:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,322 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting ad90ad7da13c4937872124748eaed24e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664092662 2024-11-26T23:34:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,334 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,336 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:54,336 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411269c01ec80fa354698904a063a8fa136f7_38491e1fe242cb24ed1e0e85404fb0f7 store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,337 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/680d7b430e96471fa65e8278234be895 is 50, key is test_row_0/B:col10/1732664093039/Put/seqid=0 2024-11-26T23:34:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,338 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411269c01ec80fa354698904a063a8fa136f7_38491e1fe242cb24ed1e0e85404fb0f7, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,338 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269c01ec80fa354698904a063a8fa136f7_38491e1fe242cb24ed1e0e85404fb0f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T23:34:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742197_1373 (size=12595) 2024-11-26T23:34:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742198_1374 (size=4469) 2024-11-26T23:34:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T23:34:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,439 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-26T23:34:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265bc4f033f3b64684a9966a05a8718e45_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664093063/Put/seqid=0 2024-11-26T23:34:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742199_1375 (size=9814) 2024-11-26T23:34:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,463 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265bc4f033f3b64684a9966a05a8718e45_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265bc4f033f3b64684a9966a05a8718e45_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2bb73396e06d4aa590650af277e48b87, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2bb73396e06d4aa590650af277e48b87 is 175, key is test_row_0/A:col10/1732664093063/Put/seqid=0 2024-11-26T23:34:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742200_1376 (size=22461) 2024-11-26T23:34:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2bb73396e06d4aa590650af277e48b87 2024-11-26T23:34:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/9006585757c14fa5999efd8aab89b78b is 50, key is test_row_0/B:col10/1732664093063/Put/seqid=0 2024-11-26T23:34:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742201_1377 (size=9757) 2024-11-26T23:34:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,507 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/9006585757c14fa5999efd8aab89b78b 2024-11-26T23:34:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/650862f25ab5414f936443b5af4468c7 is 50, key is test_row_0/C:col10/1732664093063/Put/seqid=0 2024-11-26T23:34:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742202_1378 (size=9757) 2024-11-26T23:34:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,535 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/650862f25ab5414f936443b5af4468c7 2024-11-26T23:34:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/2bb73396e06d4aa590650af277e48b87 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2bb73396e06d4aa590650af277e48b87 2024-11-26T23:34:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2bb73396e06d4aa590650af277e48b87, entries=100, sequenceid=207, filesize=21.9 K 2024-11-26T23:34:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/9006585757c14fa5999efd8aab89b78b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/9006585757c14fa5999efd8aab89b78b 2024-11-26T23:34:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,553 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/9006585757c14fa5999efd8aab89b78b, entries=100, sequenceid=207, filesize=9.5 K 2024-11-26T23:34:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/650862f25ab5414f936443b5af4468c7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/650862f25ab5414f936443b5af4468c7 2024-11-26T23:34:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,558 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/650862f25ab5414f936443b5af4468c7, entries=100, sequenceid=207, filesize=9.5 K 2024-11-26T23:34:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,560 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 38491e1fe242cb24ed1e0e85404fb0f7 in 121ms, sequenceid=207, compaction requested=true 2024-11-26T23:34:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-26T23:34:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-26T23:34:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-26T23:34:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3490 sec 2024-11-26T23:34:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.3550 sec 2024-11-26T23:34:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,756 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#A#compaction#317 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,757 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6a0ea71313ea45d7a5a3d5cbf89585cf is 175, key is test_row_0/A:col10/1732664093039/Put/seqid=0 2024-11-26T23:34:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,761 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/680d7b430e96471fa65e8278234be895 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/680d7b430e96471fa65e8278234be895 2024-11-26T23:34:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742203_1379 (size=31549) 2024-11-26T23:34:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,770 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/B of 38491e1fe242cb24ed1e0e85404fb0f7 into 680d7b430e96471fa65e8278234be895(size=12.3 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:54,770 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:54,771 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/B, priority=13, startTime=1732664094318; duration=0sec 2024-11-26T23:34:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,771 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:54,771 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:B 2024-11-26T23:34:54,771 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:34:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,773 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:34:54,773 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/C is initiating minor compaction (all files) 2024-11-26T23:34:54,773 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/C in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,773 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/fe815faa0eeb48bba820a06012d74e8f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/05c3dbbf33634156a1de193bce3d8e08, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3dcdce0058d54df5947623c5ffc6e8af, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/650862f25ab5414f936443b5af4468c7] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=43.1 K 2024-11-26T23:34:54,773 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting fe815faa0eeb48bba820a06012d74e8f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732664089246 2024-11-26T23:34:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,774 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 05c3dbbf33634156a1de193bce3d8e08, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732664090430 2024-11-26T23:34:54,774 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dcdce0058d54df5947623c5ffc6e8af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664092662 2024-11-26T23:34:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,774 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 650862f25ab5414f936443b5af4468c7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732664093063 2024-11-26T23:34:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,775 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6a0ea71313ea45d7a5a3d5cbf89585cf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6a0ea71313ea45d7a5a3d5cbf89585cf 2024-11-26T23:34:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,781 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/A of 38491e1fe242cb24ed1e0e85404fb0f7 into 6a0ea71313ea45d7a5a3d5cbf89585cf(size=30.8 K), total size for store is 52.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:54,781 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:54,781 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/A, priority=13, startTime=1732664094318; duration=0sec 2024-11-26T23:34:54,781 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:54,781 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:A 2024-11-26T23:34:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,798 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#C#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,799 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/b64040bc3ddd4a6db14805838b8ba086 is 50, key is test_row_0/C:col10/1732664093063/Put/seqid=0 2024-11-26T23:34:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742204_1380 (size=12629) 2024-11-26T23:34:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:55,084 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:34:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112680464f088a8b4546b1fe3ba4492d73b5_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664095046/Put/seqid=0 2024-11-26T23:34:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742205_1381 (size=14794) 2024-11-26T23:34:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,219 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/b64040bc3ddd4a6db14805838b8ba086 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/b64040bc3ddd4a6db14805838b8ba086 2024-11-26T23:34:55,223 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/C of 38491e1fe242cb24ed1e0e85404fb0f7 into b64040bc3ddd4a6db14805838b8ba086(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:55,223 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:55,223 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/C, priority=12, startTime=1732664094318; duration=0sec 2024-11-26T23:34:55,223 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:55,223 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:C 2024-11-26T23:34:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664155216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664155216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664155231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,243 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_0' in 38491e1fe242cb24ed1e0e85404fb0f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664155242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,247 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 38491e1fe242cb24ed1e0e85404fb0f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664155331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664155332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T23:34:55,362 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:55,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-26T23:34:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T23:34:55,364 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:55,365 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:55,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T23:34:55,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:55,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112680464f088a8b4546b1fe3ba4492d73b5_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112680464f088a8b4546b1fe3ba4492d73b5_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:55,505 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/ba8bb8bc3ac04dc198d708ff40ed4a42, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:55,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/ba8bb8bc3ac04dc198d708ff40ed4a42 is 175, key is test_row_0/A:col10/1732664095046/Put/seqid=0 2024-11-26T23:34:55,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742206_1382 (size=39749) 2024-11-26T23:34:55,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664155541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664155551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T23:34:55,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664155852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664155871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:55,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/ba8bb8bc3ac04dc198d708ff40ed4a42 2024-11-26T23:34:55,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/6bedd3aef27e4b75a340522228779639 is 50, key is test_row_0/B:col10/1732664095046/Put/seqid=0 2024-11-26T23:34:55,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742207_1383 (size=12151) 2024-11-26T23:34:55,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:55,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T23:34:56,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/6bedd3aef27e4b75a340522228779639 2024-11-26T23:34:56,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/67be666a64314703bf2e9efabcca4efc is 50, key is test_row_0/C:col10/1732664095046/Put/seqid=0 2024-11-26T23:34:56,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742208_1384 (size=12151) 2024-11-26T23:34:56,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664156372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:56,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664156392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:56,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:56,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T23:34:56,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:56,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/67be666a64314703bf2e9efabcca4efc 2024-11-26T23:34:56,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/ba8bb8bc3ac04dc198d708ff40ed4a42 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ba8bb8bc3ac04dc198d708ff40ed4a42 2024-11-26T23:34:56,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ba8bb8bc3ac04dc198d708ff40ed4a42, entries=200, sequenceid=220, filesize=38.8 K 2024-11-26T23:34:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/6bedd3aef27e4b75a340522228779639 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/6bedd3aef27e4b75a340522228779639 2024-11-26T23:34:56,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/6bedd3aef27e4b75a340522228779639, entries=150, sequenceid=220, filesize=11.9 K 2024-11-26T23:34:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/67be666a64314703bf2e9efabcca4efc as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/67be666a64314703bf2e9efabcca4efc 2024-11-26T23:34:56,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/67be666a64314703bf2e9efabcca4efc, entries=150, sequenceid=220, filesize=11.9 K 2024-11-26T23:34:56,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1666ms, sequenceid=220, compaction requested=true 2024-11-26T23:34:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:34:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:56,751 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:34:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:34:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:56,751 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:34:56,752 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:56,752 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:34:56,752 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/B is initiating minor compaction (all files) 2024-11-26T23:34:56,752 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/A is initiating minor compaction (all files) 2024-11-26T23:34:56,752 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/B in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,752 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/A in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,752 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/680d7b430e96471fa65e8278234be895, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/9006585757c14fa5999efd8aab89b78b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/6bedd3aef27e4b75a340522228779639] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=33.7 K 2024-11-26T23:34:56,752 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6a0ea71313ea45d7a5a3d5cbf89585cf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2bb73396e06d4aa590650af277e48b87, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ba8bb8bc3ac04dc198d708ff40ed4a42] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=91.6 K 2024-11-26T23:34:56,752 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,752 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6a0ea71313ea45d7a5a3d5cbf89585cf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2bb73396e06d4aa590650af277e48b87, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ba8bb8bc3ac04dc198d708ff40ed4a42] 2024-11-26T23:34:56,752 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 680d7b430e96471fa65e8278234be895, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664092662 2024-11-26T23:34:56,752 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0ea71313ea45d7a5a3d5cbf89585cf, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664092662 2024-11-26T23:34:56,753 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bb73396e06d4aa590650af277e48b87, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732664093063 2024-11-26T23:34:56,753 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9006585757c14fa5999efd8aab89b78b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732664093063 2024-11-26T23:34:56,753 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba8bb8bc3ac04dc198d708ff40ed4a42, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732664095026 2024-11-26T23:34:56,753 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bedd3aef27e4b75a340522228779639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732664095046 2024-11-26T23:34:56,759 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:56,761 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126e844e8dd1459465dadfef1ce0f01d135_38491e1fe242cb24ed1e0e85404fb0f7 store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:56,761 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:56,762 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/f2edbc4fce6d487696772a3d12a8863b is 50, key is test_row_0/B:col10/1732664095046/Put/seqid=0 2024-11-26T23:34:56,763 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126e844e8dd1459465dadfef1ce0f01d135_38491e1fe242cb24ed1e0e85404fb0f7, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:56,763 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e844e8dd1459465dadfef1ce0f01d135_38491e1fe242cb24ed1e0e85404fb0f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:56,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742209_1385 (size=12697) 2024-11-26T23:34:56,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742210_1386 (size=4469) 2024-11-26T23:34:56,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T23:34:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:56,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:34:56,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268cd6deb65e374b60a8b8ae442c7d18b6_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664095213/Put/seqid=0 2024-11-26T23:34:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742211_1387 (size=12304) 2024-11-26T23:34:57,174 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#A#compaction#326 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:34:57,174 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/0cc4a5e96ff4485d9ac0f0ec325df38b is 175, key is test_row_0/A:col10/1732664095046/Put/seqid=0 2024-11-26T23:34:57,177 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/f2edbc4fce6d487696772a3d12a8863b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f2edbc4fce6d487696772a3d12a8863b 2024-11-26T23:34:57,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742212_1388 (size=31651) 2024-11-26T23:34:57,182 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/B of 38491e1fe242cb24ed1e0e85404fb0f7 into f2edbc4fce6d487696772a3d12a8863b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:57,183 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/B, priority=13, startTime=1732664096751; duration=0sec 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:B 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. because compaction request was cancelled 2024-11-26T23:34:57,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:C 2024-11-26T23:34:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:57,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:57,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664157283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664157292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:57,312 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268cd6deb65e374b60a8b8ae442c7d18b6_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268cd6deb65e374b60a8b8ae442c7d18b6_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:57,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/f1bf9fae55314bfa95245d5ee868812a, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:57,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/f1bf9fae55314bfa95245d5ee868812a is 175, key is test_row_0/A:col10/1732664095213/Put/seqid=0 2024-11-26T23:34:57,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742213_1389 (size=31105) 2024-11-26T23:34:57,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664157391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,400 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 38491e1fe242cb24ed1e0e85404fb0f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664157401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664157412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664157412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,421 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 38491e1fe242cb24ed1e0e85404fb0f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T23:34:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1732664157532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,537 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 38491e1fe242cb24ed1e0e85404fb0f7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:34:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,583 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/0cc4a5e96ff4485d9ac0f0ec325df38b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0cc4a5e96ff4485d9ac0f0ec325df38b 2024-11-26T23:34:57,587 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/A of 38491e1fe242cb24ed1e0e85404fb0f7 into 0cc4a5e96ff4485d9ac0f0ec325df38b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:34:57,587 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:57,587 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/A, priority=13, startTime=1732664096751; duration=0sec 2024-11-26T23:34:57,587 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:34:57,587 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:A 2024-11-26T23:34:57,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664157621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664157632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,718 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/f1bf9fae55314bfa95245d5ee868812a 2024-11-26T23:34:57,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/c6f2824bf3134510a17c9e5833cebf06 is 50, key is test_row_0/B:col10/1732664095213/Put/seqid=0 2024-11-26T23:34:57,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742214_1390 (size=12151) 2024-11-26T23:34:57,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664157942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664157942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:58,130 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/c6f2824bf3134510a17c9e5833cebf06 2024-11-26T23:34:58,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/6c11975656ef42b39dd2204180ba5c24 is 50, key is test_row_0/C:col10/1732664095213/Put/seqid=0 2024-11-26T23:34:58,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742215_1391 (size=12151) 2024-11-26T23:34:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664158462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:58,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664158462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:58,541 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/6c11975656ef42b39dd2204180ba5c24 2024-11-26T23:34:58,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/f1bf9fae55314bfa95245d5ee868812a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/f1bf9fae55314bfa95245d5ee868812a 2024-11-26T23:34:58,548 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/f1bf9fae55314bfa95245d5ee868812a, entries=150, sequenceid=246, filesize=30.4 K 2024-11-26T23:34:58,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/c6f2824bf3134510a17c9e5833cebf06 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c6f2824bf3134510a17c9e5833cebf06 2024-11-26T23:34:58,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c6f2824bf3134510a17c9e5833cebf06, entries=150, sequenceid=246, filesize=11.9 K 2024-11-26T23:34:58,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/6c11975656ef42b39dd2204180ba5c24 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6c11975656ef42b39dd2204180ba5c24 2024-11-26T23:34:58,555 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6c11975656ef42b39dd2204180ba5c24, entries=150, sequenceid=246, filesize=11.9 K 2024-11-26T23:34:58,556 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1660ms, sequenceid=246, compaction requested=true 2024-11-26T23:34:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:34:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-26T23:34:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-26T23:34:58,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-26T23:34:58,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1930 sec 2024-11-26T23:34:58,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.1970 sec 2024-11-26T23:34:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:59,425 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:34:59,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T23:34:59,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e07aa4c9e9aa48c0a18a08ed6ae98595_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664097279/Put/seqid=0 2024-11-26T23:34:59,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742216_1392 (size=14894) 2024-11-26T23:34:59,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:34:59,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e07aa4c9e9aa48c0a18a08ed6ae98595_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e07aa4c9e9aa48c0a18a08ed6ae98595_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:34:59,442 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6552703bba0e4be2a0b137454f0403d4, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:34:59,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6552703bba0e4be2a0b137454f0403d4 is 175, key is test_row_0/A:col10/1732664097279/Put/seqid=0 2024-11-26T23:34:59,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742217_1393 (size=39849) 2024-11-26T23:34:59,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664159499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664159507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664159507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664159512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T23:34:59,542 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:34:59,542 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:34:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-26T23:34:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:34:59,544 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:34:59,545 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:34:59,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:34:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664159621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664159621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664159622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664159622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:34:59,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:34:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:59,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664159831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6552703bba0e4be2a0b137454f0403d4 2024-11-26T23:34:59,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664159842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664159842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:34:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664159842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:34:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:34:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:34:59,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:34:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:34:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:34:59,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:34:59,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/675c7a50a1824b26baa1f7ee289e7ceb is 50, key is test_row_0/B:col10/1732664097279/Put/seqid=0 2024-11-26T23:34:59,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742218_1394 (size=12201) 2024-11-26T23:34:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:35:00,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664160141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:00,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664160151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:00,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46844 deadline: 1732664160161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:35:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46830 deadline: 1732664160162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:00,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/675c7a50a1824b26baa1f7ee289e7ceb 2024-11-26T23:35:00,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/5cbcea022bc84ef1822631624840004d is 50, key is test_row_0/C:col10/1732664097279/Put/seqid=0 2024-11-26T23:35:00,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742219_1395 (size=12201) 2024-11-26T23:35:00,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,458 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-26T23:35:00,458 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:00,458 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,458 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:00,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:00,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,462 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-26T23:35:00,462 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:00,462 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,462 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,465 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-26T23:35:00,465 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:00,465 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,465 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:00,469 INFO [Thread-1477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1477. 2024-11-26T23:35:00,469 DEBUG [Thread-1477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:00,469 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,469 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:00,473 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-26T23:35:00,473 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:00,473 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,473 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:00,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732664160661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:00,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/5cbcea022bc84ef1822631624840004d 2024-11-26T23:35:00,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1732664160671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:00,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/6552703bba0e4be2a0b137454f0403d4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6552703bba0e4be2a0b137454f0403d4 2024-11-26T23:35:00,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6552703bba0e4be2a0b137454f0403d4, entries=200, sequenceid=260, filesize=38.9 K 2024-11-26T23:35:00,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/675c7a50a1824b26baa1f7ee289e7ceb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/675c7a50a1824b26baa1f7ee289e7ceb 2024-11-26T23:35:00,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/675c7a50a1824b26baa1f7ee289e7ceb, entries=150, sequenceid=260, filesize=11.9 K 2024-11-26T23:35:00,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/5cbcea022bc84ef1822631624840004d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5cbcea022bc84ef1822631624840004d 2024-11-26T23:35:00,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5cbcea022bc84ef1822631624840004d, entries=150, sequenceid=260, filesize=11.9 K 2024-11-26T23:35:00,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1256ms, sequenceid=260, compaction requested=true 2024-11-26T23:35:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:35:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:00,682 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:00,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:00,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:00,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38491e1fe242cb24ed1e0e85404fb0f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:00,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:00,682 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:35:00,682 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:00,682 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:00,682 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/A is initiating minor compaction (all files) 2024-11-26T23:35:00,682 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/B is initiating minor compaction (all files) 2024-11-26T23:35:00,682 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/A in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,682 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/B in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,683 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0cc4a5e96ff4485d9ac0f0ec325df38b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/f1bf9fae55314bfa95245d5ee868812a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6552703bba0e4be2a0b137454f0403d4] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=100.2 K 2024-11-26T23:35:00,683 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f2edbc4fce6d487696772a3d12a8863b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c6f2824bf3134510a17c9e5833cebf06, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/675c7a50a1824b26baa1f7ee289e7ceb] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=36.2 K 2024-11-26T23:35:00,683 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,683 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0cc4a5e96ff4485d9ac0f0ec325df38b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/f1bf9fae55314bfa95245d5ee868812a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6552703bba0e4be2a0b137454f0403d4] 2024-11-26T23:35:00,683 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f2edbc4fce6d487696772a3d12a8863b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732664095046 2024-11-26T23:35:00,683 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cc4a5e96ff4485d9ac0f0ec325df38b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732664095046 2024-11-26T23:35:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:00,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:00,683 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f2824bf3134510a17c9e5833cebf06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732664095186 2024-11-26T23:35:00,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T23:35:00,683 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1bf9fae55314bfa95245d5ee868812a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732664095186 2024-11-26T23:35:00,683 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-11-26T23:35:00,683 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-26T23:35:00,683 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:00,683 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:00,683 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,683 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,683 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 675c7a50a1824b26baa1f7ee289e7ceb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732664097279 2024-11-26T23:35:00,683 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,683 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:00,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:00,684 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6552703bba0e4be2a0b137454f0403d4, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732664097279 2024-11-26T23:35:00,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:00,690 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#B#compaction#335 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:00,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262a00857d9d2243dcb3d2c27722ea084e_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664099505/Put/seqid=0 2024-11-26T23:35:00,690 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/5c444be3522941dfa29af7ef21cc3d1a is 50, key is test_row_0/B:col10/1732664097279/Put/seqid=0 2024-11-26T23:35:00,693 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:35:00,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742221_1397 (size=12849) 2024-11-26T23:35:00,699 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264798d80a396548ebb9a6222bf87af376_38491e1fe242cb24ed1e0e85404fb0f7 store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:35:00,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742220_1396 (size=12454) 2024-11-26T23:35:00,703 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264798d80a396548ebb9a6222bf87af376_38491e1fe242cb24ed1e0e85404fb0f7, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:35:00,704 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264798d80a396548ebb9a6222bf87af376_38491e1fe242cb24ed1e0e85404fb0f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:35:00,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742222_1398 (size=4469) 2024-11-26T23:35:00,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:00,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:00,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,099 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/5c444be3522941dfa29af7ef21cc3d1a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5c444be3522941dfa29af7ef21cc3d1a 2024-11-26T23:35:01,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:01,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262a00857d9d2243dcb3d2c27722ea084e_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262a00857d9d2243dcb3d2c27722ea084e_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:01,103 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/B of 38491e1fe242cb24ed1e0e85404fb0f7 into 5c444be3522941dfa29af7ef21cc3d1a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:01,103 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:35:01,103 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/B, priority=13, startTime=1732664100681; duration=0sec 2024-11-26T23:35:01,103 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:01,103 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:B 2024-11-26T23:35:01,103 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/e32ccc21a7d8462baf02af1f42cc2891, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:35:01,103 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:01,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/e32ccc21a7d8462baf02af1f42cc2891 is 175, key is test_row_0/A:col10/1732664099505/Put/seqid=0 2024-11-26T23:35:01,104 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:01,104 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 38491e1fe242cb24ed1e0e85404fb0f7/C is initiating minor compaction (all files) 2024-11-26T23:35:01,104 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38491e1fe242cb24ed1e0e85404fb0f7/C in TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,104 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/b64040bc3ddd4a6db14805838b8ba086, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/67be666a64314703bf2e9efabcca4efc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6c11975656ef42b39dd2204180ba5c24, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5cbcea022bc84ef1822631624840004d] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp, totalSize=48.0 K 2024-11-26T23:35:01,105 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b64040bc3ddd4a6db14805838b8ba086, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732664092662 2024-11-26T23:35:01,105 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 67be666a64314703bf2e9efabcca4efc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732664095046 2024-11-26T23:35:01,105 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c11975656ef42b39dd2204180ba5c24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732664095186 2024-11-26T23:35:01,105 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cbcea022bc84ef1822631624840004d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732664097279 2024-11-26T23:35:01,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742223_1399 (size=31255) 2024-11-26T23:35:01,109 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#A#compaction#336 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:01,109 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/a60c9016398e4f69a2a302a987d11f65 is 175, key is test_row_0/A:col10/1732664097279/Put/seqid=0 2024-11-26T23:35:01,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742224_1400 (size=31803) 2024-11-26T23:35:01,113 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38491e1fe242cb24ed1e0e85404fb0f7#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:01,113 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/6de31d528c3143d6874ea74aec1cf9d5 is 50, key is test_row_0/C:col10/1732664097279/Put/seqid=0 2024-11-26T23:35:01,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742225_1401 (size=12815) 2024-11-26T23:35:01,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,508 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/e32ccc21a7d8462baf02af1f42cc2891 2024-11-26T23:35:01,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/c29a9ab0f5bf41eb900dcbe3b5d51ef9 is 50, key is test_row_0/B:col10/1732664099505/Put/seqid=0 2024-11-26T23:35:01,516 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/a60c9016398e4f69a2a302a987d11f65 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a60c9016398e4f69a2a302a987d11f65 2024-11-26T23:35:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742226_1402 (size=12301) 2024-11-26T23:35:01,519 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/6de31d528c3143d6874ea74aec1cf9d5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6de31d528c3143d6874ea74aec1cf9d5 2024-11-26T23:35:01,520 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/A of 38491e1fe242cb24ed1e0e85404fb0f7 into a60c9016398e4f69a2a302a987d11f65(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:01,520 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:35:01,520 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/A, priority=13, startTime=1732664100681; duration=0sec 2024-11-26T23:35:01,521 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:01,521 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:A 2024-11-26T23:35:01,523 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38491e1fe242cb24ed1e0e85404fb0f7/C of 38491e1fe242cb24ed1e0e85404fb0f7 into 6de31d528c3143d6874ea74aec1cf9d5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:01,523 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:35:01,523 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., storeName=38491e1fe242cb24ed1e0e85404fb0f7/C, priority=12, startTime=1732664100682; duration=0sec 2024-11-26T23:35:01,523 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:01,523 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38491e1fe242cb24ed1e0e85404fb0f7:C 2024-11-26T23:35:01,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,673 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-26T23:35:01,673 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:01,673 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:01,673 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:01,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:01,682 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-11-26T23:35:01,683 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:01,683 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:01,683 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:01,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:01,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:35:01,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/c29a9ab0f5bf41eb900dcbe3b5d51ef9 2024-11-26T23:35:01,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/01fd569f118840ca9469e5d998bd000e is 50, key is test_row_0/C:col10/1732664099505/Put/seqid=0 2024-11-26T23:35:01,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742227_1403 (size=12301) 2024-11-26T23:35:01,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:01,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:02,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:02,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:02,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. as already flushing 2024-11-26T23:35:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:02,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:02,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/01fd569f118840ca9469e5d998bd000e 2024-11-26T23:35:02,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/e32ccc21a7d8462baf02af1f42cc2891 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/e32ccc21a7d8462baf02af1f42cc2891 2024-11-26T23:35:02,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/e32ccc21a7d8462baf02af1f42cc2891, entries=150, sequenceid=286, filesize=30.5 K 2024-11-26T23:35:02,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/c29a9ab0f5bf41eb900dcbe3b5d51ef9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c29a9ab0f5bf41eb900dcbe3b5d51ef9 2024-11-26T23:35:02,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c29a9ab0f5bf41eb900dcbe3b5d51ef9, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T23:35:02,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/01fd569f118840ca9469e5d998bd000e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/01fd569f118840ca9469e5d998bd000e 2024-11-26T23:35:02,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/01fd569f118840ca9469e5d998bd000e, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T23:35:02,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=13.42 KB/13740 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1661ms, sequenceid=286, compaction requested=false 2024-11-26T23:35:02,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:35:02,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T23:35:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:02,449 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-26T23:35:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cb6c1af9f3234a33b55a1b2091ea80e3_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_0/A:col10/1732664101682/Put/seqid=0 2024-11-26T23:35:02,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742228_1404 (size=9914) 2024-11-26T23:35:02,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:02,863 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cb6c1af9f3234a33b55a1b2091ea80e3_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cb6c1af9f3234a33b55a1b2091ea80e3_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/7d7cdb4b7ed648c2bc927bf2df90d3f9, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:35:02,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/7d7cdb4b7ed648c2bc927bf2df90d3f9 is 175, key is test_row_0/A:col10/1732664101682/Put/seqid=0 2024-11-26T23:35:02,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742229_1405 (size=22561) 2024-11-26T23:35:03,269 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/7d7cdb4b7ed648c2bc927bf2df90d3f9 2024-11-26T23:35:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/89e3755456294be4b4dfe4b7c578b78d is 50, key is test_row_0/B:col10/1732664101682/Put/seqid=0 2024-11-26T23:35:03,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742230_1406 (size=9857) 2024-11-26T23:35:03,678 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/89e3755456294be4b4dfe4b7c578b78d 2024-11-26T23:35:03,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/446d81ef2ab340ecabdac7b536cd6be3 is 50, key is test_row_0/C:col10/1732664101682/Put/seqid=0 2024-11-26T23:35:03,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742231_1407 (size=9857) 2024-11-26T23:35:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:35:04,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/446d81ef2ab340ecabdac7b536cd6be3 2024-11-26T23:35:04,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/7d7cdb4b7ed648c2bc927bf2df90d3f9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/7d7cdb4b7ed648c2bc927bf2df90d3f9 2024-11-26T23:35:04,094 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/7d7cdb4b7ed648c2bc927bf2df90d3f9, entries=100, sequenceid=294, filesize=22.0 K 2024-11-26T23:35:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/89e3755456294be4b4dfe4b7c578b78d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/89e3755456294be4b4dfe4b7c578b78d 2024-11-26T23:35:04,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/89e3755456294be4b4dfe4b7c578b78d, entries=100, sequenceid=294, filesize=9.6 K 2024-11-26T23:35:04,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/446d81ef2ab340ecabdac7b536cd6be3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/446d81ef2ab340ecabdac7b536cd6be3 2024-11-26T23:35:04,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/446d81ef2ab340ecabdac7b536cd6be3, entries=100, sequenceid=294, filesize=9.6 K 2024-11-26T23:35:04,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1653ms, sequenceid=294, compaction requested=true 2024-11-26T23:35:04,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 38491e1fe242cb24ed1e0e85404fb0f7: 2024-11-26T23:35:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-26T23:35:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-26T23:35:04,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-26T23:35:04,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5590 sec 2024-11-26T23:35:04,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 4.5630 sec 2024-11-26T23:35:04,597 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:35:07,603 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-11-26T23:35:07,603 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:07,603 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:07,604 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:07,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T23:35:07,742 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1162 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3484 rows 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3505 rows 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1164 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3485 rows 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1164 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3490 rows 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1173 2024-11-26T23:35:07,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3515 rows 2024-11-26T23:35:07,743 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:35:07,743 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:35:07,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:07,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:07,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:07,744 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T23:35:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T23:35:07,748 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664107748"}]},"ts":"1732664107748"} 2024-11-26T23:35:07,750 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T23:35:07,750 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T23:35:07,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:35:07,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, UNASSIGN}] 2024-11-26T23:35:07,754 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, UNASSIGN 2024-11-26T23:35:07,755 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:07,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, UNASSIGN because future has completed 2024-11-26T23:35:07,756 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:35:07,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:35:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T23:35:07,909 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:07,909 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:35:07,909 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 38491e1fe242cb24ed1e0e85404fb0f7, disabling compactions & flushes 2024-11-26T23:35:07,909 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:07,910 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:07,910 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. after waiting 0 ms 2024-11-26T23:35:07,910 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:07,910 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 38491e1fe242cb24ed1e0e85404fb0f7 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-26T23:35:07,915 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112632d9dd5ddb2841f3a1032bb72f668e27_38491e1fe242cb24ed1e0e85404fb0f7 is 50, key is test_row_1/A:col10/1732664107602/Put/seqid=0 2024-11-26T23:35:07,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742232_1408 (size=7374) 2024-11-26T23:35:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T23:35:08,319 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:08,323 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112632d9dd5ddb2841f3a1032bb72f668e27_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112632d9dd5ddb2841f3a1032bb72f668e27_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:08,323 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/056a566b5314440a890bc2d5552ed528, store: [table=TestAcidGuarantees family=A region=38491e1fe242cb24ed1e0e85404fb0f7] 2024-11-26T23:35:08,324 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/056a566b5314440a890bc2d5552ed528 is 175, key is test_row_1/A:col10/1732664107602/Put/seqid=0 2024-11-26T23:35:08,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742233_1409 (size=13865) 2024-11-26T23:35:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T23:35:08,727 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/056a566b5314440a890bc2d5552ed528 2024-11-26T23:35:08,733 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/899129cbc243492ba3c92963c43d28fd is 50, key is test_row_1/B:col10/1732664107602/Put/seqid=0 2024-11-26T23:35:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742234_1410 (size=7415) 2024-11-26T23:35:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T23:35:09,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/704ccf06f89a4b76ac2c4fca4f572b71, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/adb21bbf9e354cd7a727e24080b9dcfd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/717da68741014308ad3d28575cb1e099, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ccad19a4d5784d04af951467a8a12915, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0c8a097f09e0414b88296265ec4f1778, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/52f8d242941d4a0c96e2e1854c9f3e56, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/814be5d1fd2d40a096e8032c86ce5f01, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/203a5ab149d9417d9e4d7ab2df8d4194, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2beac782bf8441ea9e801151a3062dd6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a3b79807f39147d9af0497ec8959ab32, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/9fd3cf79a1214526bea98828d89f6222, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6e18772f7cd34240bfc7364f413780e0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/fa59b2d621ae4d159ae952b134772c22, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6a0ea71313ea45d7a5a3d5cbf89585cf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2bb73396e06d4aa590650af277e48b87, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ba8bb8bc3ac04dc198d708ff40ed4a42, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0cc4a5e96ff4485d9ac0f0ec325df38b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/f1bf9fae55314bfa95245d5ee868812a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6552703bba0e4be2a0b137454f0403d4] to archive 2024-11-26T23:35:09,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:35:09,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/704ccf06f89a4b76ac2c4fca4f572b71 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/704ccf06f89a4b76ac2c4fca4f572b71 2024-11-26T23:35:09,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/adb21bbf9e354cd7a727e24080b9dcfd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/adb21bbf9e354cd7a727e24080b9dcfd 2024-11-26T23:35:09,015 WARN [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-26T23:35:09,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/717da68741014308ad3d28575cb1e099 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/717da68741014308ad3d28575cb1e099 2024-11-26T23:35:09,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ccad19a4d5784d04af951467a8a12915 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ccad19a4d5784d04af951467a8a12915 2024-11-26T23:35:09,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0c8a097f09e0414b88296265ec4f1778 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0c8a097f09e0414b88296265ec4f1778 2024-11-26T23:35:09,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/52f8d242941d4a0c96e2e1854c9f3e56 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/52f8d242941d4a0c96e2e1854c9f3e56 2024-11-26T23:35:09,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/814be5d1fd2d40a096e8032c86ce5f01 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/814be5d1fd2d40a096e8032c86ce5f01 2024-11-26T23:35:09,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/203a5ab149d9417d9e4d7ab2df8d4194 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/203a5ab149d9417d9e4d7ab2df8d4194 2024-11-26T23:35:09,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2beac782bf8441ea9e801151a3062dd6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2beac782bf8441ea9e801151a3062dd6 2024-11-26T23:35:09,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a3b79807f39147d9af0497ec8959ab32 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a3b79807f39147d9af0497ec8959ab32 2024-11-26T23:35:09,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/9fd3cf79a1214526bea98828d89f6222 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/9fd3cf79a1214526bea98828d89f6222 2024-11-26T23:35:09,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6e18772f7cd34240bfc7364f413780e0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6e18772f7cd34240bfc7364f413780e0 2024-11-26T23:35:09,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/fa59b2d621ae4d159ae952b134772c22 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/fa59b2d621ae4d159ae952b134772c22 2024-11-26T23:35:09,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6a0ea71313ea45d7a5a3d5cbf89585cf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6a0ea71313ea45d7a5a3d5cbf89585cf 2024-11-26T23:35:09,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2bb73396e06d4aa590650af277e48b87 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/2bb73396e06d4aa590650af277e48b87 2024-11-26T23:35:09,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ba8bb8bc3ac04dc198d708ff40ed4a42 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/ba8bb8bc3ac04dc198d708ff40ed4a42 2024-11-26T23:35:09,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0cc4a5e96ff4485d9ac0f0ec325df38b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/0cc4a5e96ff4485d9ac0f0ec325df38b 2024-11-26T23:35:09,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/f1bf9fae55314bfa95245d5ee868812a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/f1bf9fae55314bfa95245d5ee868812a 2024-11-26T23:35:09,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6552703bba0e4be2a0b137454f0403d4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/6552703bba0e4be2a0b137454f0403d4 2024-11-26T23:35:09,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/0fc1251dbc1f4045bf03daaf440001c6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/76055f727e934aa5bad59263b7be43f7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f3af40cb010c4b6c854caea27bfdb08f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/b37c3f75b9fc4e11a85c8e02baa92358, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/7c7be85a189e492f8257eb7965bf5aab, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/4e9710b45ace43b4bf5d793e94635fda, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/bd44bc3ba08e4d4289a7a1001f57ad9a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5a88932207ae49a2a9839f9bbf6771bd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/64ed3e8c414d4b259e02f1d306c0b6e5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/67d349a0653e460697a98e58ea122297, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/8858a32803b3498a9c4d1eba6fd1a97a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/98369526f34f4817899e9c0f5e154f87, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/680d7b430e96471fa65e8278234be895, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/ad90ad7da13c4937872124748eaed24e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/9006585757c14fa5999efd8aab89b78b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f2edbc4fce6d487696772a3d12a8863b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/6bedd3aef27e4b75a340522228779639, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c6f2824bf3134510a17c9e5833cebf06, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/675c7a50a1824b26baa1f7ee289e7ceb] to archive 2024-11-26T23:35:09,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:35:09,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/0fc1251dbc1f4045bf03daaf440001c6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/0fc1251dbc1f4045bf03daaf440001c6 2024-11-26T23:35:09,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/76055f727e934aa5bad59263b7be43f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/76055f727e934aa5bad59263b7be43f7 2024-11-26T23:35:09,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f3af40cb010c4b6c854caea27bfdb08f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f3af40cb010c4b6c854caea27bfdb08f 2024-11-26T23:35:09,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/b37c3f75b9fc4e11a85c8e02baa92358 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/b37c3f75b9fc4e11a85c8e02baa92358 2024-11-26T23:35:09,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/7c7be85a189e492f8257eb7965bf5aab to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/7c7be85a189e492f8257eb7965bf5aab 2024-11-26T23:35:09,037 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/4e9710b45ace43b4bf5d793e94635fda to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/4e9710b45ace43b4bf5d793e94635fda 2024-11-26T23:35:09,038 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/bd44bc3ba08e4d4289a7a1001f57ad9a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/bd44bc3ba08e4d4289a7a1001f57ad9a 2024-11-26T23:35:09,039 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5a88932207ae49a2a9839f9bbf6771bd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5a88932207ae49a2a9839f9bbf6771bd 2024-11-26T23:35:09,039 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/64ed3e8c414d4b259e02f1d306c0b6e5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/64ed3e8c414d4b259e02f1d306c0b6e5 2024-11-26T23:35:09,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/67d349a0653e460697a98e58ea122297 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/67d349a0653e460697a98e58ea122297 2024-11-26T23:35:09,041 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/8858a32803b3498a9c4d1eba6fd1a97a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/8858a32803b3498a9c4d1eba6fd1a97a 2024-11-26T23:35:09,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/98369526f34f4817899e9c0f5e154f87 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/98369526f34f4817899e9c0f5e154f87 2024-11-26T23:35:09,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/680d7b430e96471fa65e8278234be895 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/680d7b430e96471fa65e8278234be895 2024-11-26T23:35:09,043 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/ad90ad7da13c4937872124748eaed24e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/ad90ad7da13c4937872124748eaed24e 2024-11-26T23:35:09,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/9006585757c14fa5999efd8aab89b78b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/9006585757c14fa5999efd8aab89b78b 2024-11-26T23:35:09,045 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f2edbc4fce6d487696772a3d12a8863b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/f2edbc4fce6d487696772a3d12a8863b 2024-11-26T23:35:09,046 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/6bedd3aef27e4b75a340522228779639 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/6bedd3aef27e4b75a340522228779639 2024-11-26T23:35:09,046 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c6f2824bf3134510a17c9e5833cebf06 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c6f2824bf3134510a17c9e5833cebf06 2024-11-26T23:35:09,047 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/675c7a50a1824b26baa1f7ee289e7ceb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/675c7a50a1824b26baa1f7ee289e7ceb 2024-11-26T23:35:09,051 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2854896927a74dc49775230c121e9090, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/aa621c1a28954e7483bc45cfdc23b4fb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/56770bdca20f4f55982c60ff4764cedf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5b17f9e130f940dca239b4ad6b799cf5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/04d7634b8068419facbfbfd374fadac0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/afd467c593b843cf8dd995a130b71059, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/7b157572ac704714ae4eafabf5141cd3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3c7355607fbb45afaa23f5866c8b9951, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/d6168be381bf4f64939efd38ca1f396a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/fe815faa0eeb48bba820a06012d74e8f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/89ec5328434d4a988b19cd566edead59, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/05c3dbbf33634156a1de193bce3d8e08, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3dcdce0058d54df5947623c5ffc6e8af, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/b64040bc3ddd4a6db14805838b8ba086, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/650862f25ab5414f936443b5af4468c7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/67be666a64314703bf2e9efabcca4efc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6c11975656ef42b39dd2204180ba5c24, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5cbcea022bc84ef1822631624840004d] to archive 2024-11-26T23:35:09,051 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:35:09,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2854896927a74dc49775230c121e9090 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2854896927a74dc49775230c121e9090 2024-11-26T23:35:09,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/aa621c1a28954e7483bc45cfdc23b4fb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/aa621c1a28954e7483bc45cfdc23b4fb 2024-11-26T23:35:09,054 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/56770bdca20f4f55982c60ff4764cedf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/56770bdca20f4f55982c60ff4764cedf 2024-11-26T23:35:09,055 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5b17f9e130f940dca239b4ad6b799cf5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5b17f9e130f940dca239b4ad6b799cf5 2024-11-26T23:35:09,056 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/04d7634b8068419facbfbfd374fadac0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/04d7634b8068419facbfbfd374fadac0 2024-11-26T23:35:09,057 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/afd467c593b843cf8dd995a130b71059 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/afd467c593b843cf8dd995a130b71059 2024-11-26T23:35:09,058 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/7b157572ac704714ae4eafabf5141cd3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/7b157572ac704714ae4eafabf5141cd3 2024-11-26T23:35:09,059 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3c7355607fbb45afaa23f5866c8b9951 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3c7355607fbb45afaa23f5866c8b9951 2024-11-26T23:35:09,060 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/d6168be381bf4f64939efd38ca1f396a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/d6168be381bf4f64939efd38ca1f396a 2024-11-26T23:35:09,061 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/fe815faa0eeb48bba820a06012d74e8f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/fe815faa0eeb48bba820a06012d74e8f 2024-11-26T23:35:09,062 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/89ec5328434d4a988b19cd566edead59 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/89ec5328434d4a988b19cd566edead59 2024-11-26T23:35:09,062 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/05c3dbbf33634156a1de193bce3d8e08 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/05c3dbbf33634156a1de193bce3d8e08 2024-11-26T23:35:09,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3dcdce0058d54df5947623c5ffc6e8af to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/3dcdce0058d54df5947623c5ffc6e8af 2024-11-26T23:35:09,064 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/b64040bc3ddd4a6db14805838b8ba086 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/b64040bc3ddd4a6db14805838b8ba086 2024-11-26T23:35:09,065 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/650862f25ab5414f936443b5af4468c7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/650862f25ab5414f936443b5af4468c7 2024-11-26T23:35:09,066 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/67be666a64314703bf2e9efabcca4efc to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/67be666a64314703bf2e9efabcca4efc 2024-11-26T23:35:09,067 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6c11975656ef42b39dd2204180ba5c24 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6c11975656ef42b39dd2204180ba5c24 2024-11-26T23:35:09,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5cbcea022bc84ef1822631624840004d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/5cbcea022bc84ef1822631624840004d 2024-11-26T23:35:09,137 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/899129cbc243492ba3c92963c43d28fd 2024-11-26T23:35:09,143 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/2dd192fab63c42d99d2485e844c1d0e7 is 50, key is test_row_1/C:col10/1732664107602/Put/seqid=0 2024-11-26T23:35:09,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742235_1411 (size=7415) 2024-11-26T23:35:09,546 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/2dd192fab63c42d99d2485e844c1d0e7 2024-11-26T23:35:09,550 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/A/056a566b5314440a890bc2d5552ed528 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/056a566b5314440a890bc2d5552ed528 2024-11-26T23:35:09,553 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/056a566b5314440a890bc2d5552ed528, entries=50, sequenceid=298, filesize=13.5 K 2024-11-26T23:35:09,554 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/B/899129cbc243492ba3c92963c43d28fd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/899129cbc243492ba3c92963c43d28fd 2024-11-26T23:35:09,557 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/899129cbc243492ba3c92963c43d28fd, entries=50, sequenceid=298, filesize=7.2 K 2024-11-26T23:35:09,558 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/.tmp/C/2dd192fab63c42d99d2485e844c1d0e7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2dd192fab63c42d99d2485e844c1d0e7 2024-11-26T23:35:09,561 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2dd192fab63c42d99d2485e844c1d0e7, entries=50, sequenceid=298, filesize=7.2 K 2024-11-26T23:35:09,562 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1652ms, sequenceid=298, compaction requested=true 2024-11-26T23:35:09,566 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/recovered.edits/301.seqid, newMaxSeqId=301, maxSeqId=4 2024-11-26T23:35:09,567 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. 2024-11-26T23:35:09,567 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 38491e1fe242cb24ed1e0e85404fb0f7: Waiting for close lock at 1732664107909Running coprocessor pre-close hooks at 1732664107909Disabling compacts and flushes for region at 1732664107909Disabling writes for close at 1732664107910 (+1 ms)Obtaining lock to block concurrent updates at 1732664107910Preparing flush snapshotting stores in 38491e1fe242cb24ed1e0e85404fb0f7 at 1732664107910Finished memstore snapshotting TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732664107910Flushing stores of TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7. at 1732664107910Flushing 38491e1fe242cb24ed1e0e85404fb0f7/A: creating writer at 1732664107911 (+1 ms)Flushing mob file 38491e1fe242cb24ed1e0e85404fb0f7/A: appending metadata at 1732664107914 (+3 ms)Flushing mob file 38491e1fe242cb24ed1e0e85404fb0f7/A: closing flushed file at 1732664107915 (+1 ms)Flushing 38491e1fe242cb24ed1e0e85404fb0f7/A: appending metadata at 1732664108323 (+408 ms)Flushing 38491e1fe242cb24ed1e0e85404fb0f7/A: closing flushed file at 1732664108323Flushing 38491e1fe242cb24ed1e0e85404fb0f7/B: creating writer at 1732664108731 (+408 ms)Flushing 38491e1fe242cb24ed1e0e85404fb0f7/B: appending metadata at 1732664108733 (+2 ms)Flushing 38491e1fe242cb24ed1e0e85404fb0f7/B: closing flushed file at 1732664108733Flushing 38491e1fe242cb24ed1e0e85404fb0f7/C: creating writer at 1732664109140 (+407 ms)Flushing 38491e1fe242cb24ed1e0e85404fb0f7/C: appending metadata at 1732664109142 (+2 ms)Flushing 38491e1fe242cb24ed1e0e85404fb0f7/C: closing flushed file at 1732664109142Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e7127cb: reopening flushed file at 1732664109550 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6162112c: reopening flushed file at 1732664109553 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33be845f: reopening flushed file at 1732664109557 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 38491e1fe242cb24ed1e0e85404fb0f7 in 1652ms, sequenceid=298, compaction requested=true at 1732664109562 (+5 ms)Writing region close event to WAL at 1732664109564 (+2 ms)Running coprocessor post-close hooks at 1732664109567 (+3 ms)Closed at 1732664109567 2024-11-26T23:35:09,569 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,569 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=38491e1fe242cb24ed1e0e85404fb0f7, regionState=CLOSED 2024-11-26T23:35:09,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:35:09,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-26T23:35:09,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 38491e1fe242cb24ed1e0e85404fb0f7, server=c707e9ace675,43077,1732663986534 in 1.8150 sec 2024-11-26T23:35:09,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-26T23:35:09,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38491e1fe242cb24ed1e0e85404fb0f7, UNASSIGN in 1.8200 sec 2024-11-26T23:35:09,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-26T23:35:09,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8250 sec 2024-11-26T23:35:09,577 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664109577"}]},"ts":"1732664109577"} 2024-11-26T23:35:09,578 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T23:35:09,578 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T23:35:09,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8350 sec 2024-11-26T23:35:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T23:35:09,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:09,892 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T23:35:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:09,894 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T23:35:09,894 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:09,896 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,897 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/recovered.edits] 2024-11-26T23:35:09,899 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/056a566b5314440a890bc2d5552ed528 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/056a566b5314440a890bc2d5552ed528 2024-11-26T23:35:09,900 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/7d7cdb4b7ed648c2bc927bf2df90d3f9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/7d7cdb4b7ed648c2bc927bf2df90d3f9 2024-11-26T23:35:09,901 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a60c9016398e4f69a2a302a987d11f65 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/a60c9016398e4f69a2a302a987d11f65 2024-11-26T23:35:09,902 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/e32ccc21a7d8462baf02af1f42cc2891 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/A/e32ccc21a7d8462baf02af1f42cc2891 2024-11-26T23:35:09,904 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5c444be3522941dfa29af7ef21cc3d1a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/5c444be3522941dfa29af7ef21cc3d1a 2024-11-26T23:35:09,905 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/899129cbc243492ba3c92963c43d28fd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/899129cbc243492ba3c92963c43d28fd 2024-11-26T23:35:09,905 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/89e3755456294be4b4dfe4b7c578b78d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/89e3755456294be4b4dfe4b7c578b78d 2024-11-26T23:35:09,906 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c29a9ab0f5bf41eb900dcbe3b5d51ef9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/B/c29a9ab0f5bf41eb900dcbe3b5d51ef9 2024-11-26T23:35:09,908 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/01fd569f118840ca9469e5d998bd000e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/01fd569f118840ca9469e5d998bd000e 2024-11-26T23:35:09,909 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2dd192fab63c42d99d2485e844c1d0e7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/2dd192fab63c42d99d2485e844c1d0e7 2024-11-26T23:35:09,909 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/446d81ef2ab340ecabdac7b536cd6be3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/446d81ef2ab340ecabdac7b536cd6be3 2024-11-26T23:35:09,910 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6de31d528c3143d6874ea74aec1cf9d5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/C/6de31d528c3143d6874ea74aec1cf9d5 2024-11-26T23:35:09,912 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/recovered.edits/301.seqid to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7/recovered.edits/301.seqid 2024-11-26T23:35:09,912 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,912 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T23:35:09,913 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T23:35:09,913 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T23:35:09,915 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262a00857d9d2243dcb3d2c27722ea084e_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262a00857d9d2243dcb3d2c27722ea084e_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,916 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112632d9dd5ddb2841f3a1032bb72f668e27_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112632d9dd5ddb2841f3a1032bb72f668e27_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,917 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112636c6b5c717b94d5cacf53143b3df2a86_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112636c6b5c717b94d5cacf53143b3df2a86_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,918 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112637009f437e4b46508bd16dbea0d9f89a_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112637009f437e4b46508bd16dbea0d9f89a_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,919 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265af36ede07b24b60bbbb32919da48098_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265af36ede07b24b60bbbb32919da48098_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,920 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265bc4f033f3b64684a9966a05a8718e45_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265bc4f033f3b64684a9966a05a8718e45_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,920 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265e306deb67b64982b220865a49332ab8_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265e306deb67b64982b220865a49332ab8_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,921 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266511f0010bda4864a5b4c40b77584b5a_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266511f0010bda4864a5b4c40b77584b5a_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,922 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112680464f088a8b4546b1fe3ba4492d73b5_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112680464f088a8b4546b1fe3ba4492d73b5_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,923 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268cd6deb65e374b60a8b8ae442c7d18b6_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268cd6deb65e374b60a8b8ae442c7d18b6_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,924 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269e75a8d4a21f4572bae38727b294df82_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269e75a8d4a21f4572bae38727b294df82_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,925 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126afe0dcd103a04a16be26c952c490aaad_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126afe0dcd103a04a16be26c952c490aaad_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,926 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cb6c1af9f3234a33b55a1b2091ea80e3_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cb6c1af9f3234a33b55a1b2091ea80e3_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,927 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d9c44c64ef52431e975029521f3f40f9_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d9c44c64ef52431e975029521f3f40f9_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,928 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126db721c53744d4c7180fefccbc4e40199_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126db721c53744d4c7180fefccbc4e40199_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,929 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e07aa4c9e9aa48c0a18a08ed6ae98595_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e07aa4c9e9aa48c0a18a08ed6ae98595_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,930 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fe20c1cf817a4d0a86be7494cc3103eb_38491e1fe242cb24ed1e0e85404fb0f7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fe20c1cf817a4d0a86be7494cc3103eb_38491e1fe242cb24ed1e0e85404fb0f7 2024-11-26T23:35:09,930 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T23:35:09,933 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:09,935 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T23:35:09,936 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T23:35:09,937 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:09,937 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T23:35:09,938 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732664109937"}]},"ts":"9223372036854775807"} 2024-11-26T23:35:09,939 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T23:35:09,939 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 38491e1fe242cb24ed1e0e85404fb0f7, NAME => 'TestAcidGuarantees,,1732664077340.38491e1fe242cb24ed1e0e85404fb0f7.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T23:35:09,939 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T23:35:09,939 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732664109939"}]},"ts":"9223372036854775807"} 2024-11-26T23:35:09,941 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T23:35:09,941 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:09,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-26T23:35:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T23:35:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T23:35:10,002 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:10,012 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 240), OpenFileDescriptor=457 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=321 (was 351), ProcessCount=11 (was 11), AvailableMemoryMB=6454 (was 6540) 2024-11-26T23:35:10,020 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=6454 2024-11-26T23:35:10,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:35:10,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:35:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:10,023 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:35:10,024 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:10,024 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-26T23:35:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-26T23:35:10,024 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:35:10,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742236_1412 (size=959) 2024-11-26T23:35:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-26T23:35:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-26T23:35:10,431 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:35:10,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742237_1413 (size=53) 2024-11-26T23:35:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-26T23:35:10,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:35:10,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 88d8390f4cce6ec80971092bc0ed5e72, disabling compactions & flushes 2024-11-26T23:35:10,839 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:10,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:10,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. after waiting 0 ms 2024-11-26T23:35:10,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:10,839 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:10,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88d8390f4cce6ec80971092bc0ed5e72: Waiting for close lock at 1732664110839Disabling compacts and flushes for region at 1732664110839Disabling writes for close at 1732664110839Writing region close event to WAL at 1732664110839Closed at 1732664110839 2024-11-26T23:35:10,840 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:35:10,840 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732664110840"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732664110840"}]},"ts":"1732664110840"} 2024-11-26T23:35:10,841 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:35:10,842 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:35:10,842 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664110842"}]},"ts":"1732664110842"} 2024-11-26T23:35:10,843 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T23:35:10,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, ASSIGN}] 2024-11-26T23:35:10,844 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, ASSIGN 2024-11-26T23:35:10,845 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, ASSIGN; state=OFFLINE, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=false 2024-11-26T23:35:10,996 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=88d8390f4cce6ec80971092bc0ed5e72, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:10,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, ASSIGN because future has completed 2024-11-26T23:35:10,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:35:11,152 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:11,152 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:35:11,153 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,153 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:35:11,153 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,153 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,154 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,155 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88d8390f4cce6ec80971092bc0ed5e72 columnFamilyName A 2024-11-26T23:35:11,155 DEBUG [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:11,155 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] regionserver.HStore(327): Store=88d8390f4cce6ec80971092bc0ed5e72/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:11,155 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,156 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88d8390f4cce6ec80971092bc0ed5e72 columnFamilyName B 2024-11-26T23:35:11,156 DEBUG [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:11,157 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] regionserver.HStore(327): Store=88d8390f4cce6ec80971092bc0ed5e72/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:11,157 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,157 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88d8390f4cce6ec80971092bc0ed5e72 columnFamilyName C 2024-11-26T23:35:11,157 DEBUG [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:11,158 INFO [StoreOpener-88d8390f4cce6ec80971092bc0ed5e72-1 {}] regionserver.HStore(327): Store=88d8390f4cce6ec80971092bc0ed5e72/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:11,158 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,158 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,158 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,159 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,159 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,159 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:35:11,160 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-26T23:35:11,162 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:35:11,162 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 88d8390f4cce6ec80971092bc0ed5e72; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69289299, jitterRate=0.032491013407707214}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:35:11,162 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:11,163 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 88d8390f4cce6ec80971092bc0ed5e72: Running coprocessor pre-open hook at 1732664111153Writing region info on filesystem at 1732664111153Initializing all the Stores at 1732664111153Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664111153Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664111154 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664111154Cleaning up temporary data from old regions at 1732664111159 (+5 ms)Running coprocessor post-open hooks at 1732664111162 (+3 ms)Region opened successfully at 1732664111163 (+1 ms) 2024-11-26T23:35:11,163 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., pid=116, masterSystemTime=1732664111149 2024-11-26T23:35:11,165 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:11,165 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:11,165 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=88d8390f4cce6ec80971092bc0ed5e72, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:11,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:35:11,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-26T23:35:11,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 in 169 msec 2024-11-26T23:35:11,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-26T23:35:11,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, ASSIGN in 324 msec 2024-11-26T23:35:11,170 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:35:11,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664111170"}]},"ts":"1732664111170"} 2024-11-26T23:35:11,172 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T23:35:11,172 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:35:11,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-26T23:35:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-26T23:35:12,172 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:12,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,174 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330068f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,175 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a964ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,176 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,178 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:35:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35833,1732663985624 2024-11-26T23:35:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@520b0050 2024-11-26T23:35:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:35:12,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:35:12,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,182 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c517a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,183 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f16c7ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,184 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@768c08a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6aebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d841c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe56f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26efe6bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,191 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c582a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7165e4f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,193 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6046e48d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,194 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c8412c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,195 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5961d376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,197 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bdc91f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,198 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b02b7cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,199 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd95893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,200 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4551e113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,202 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367d3bb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,202 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6949eb61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:12,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:12,204 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57368a9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:12,205 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ec989b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:12,206 DEBUG [Thread-1872 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,206 DEBUG [Thread-1874 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,206 DEBUG [Thread-1889 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,206 DEBUG [Thread-1881 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,206 DEBUG [Thread-1887 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,206 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,207 DEBUG [Thread-1876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,207 DEBUG [Thread-1878 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,209 DEBUG [Thread-1883 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,209 DEBUG [Thread-1885 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:12,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:12,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,211 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:12,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-26T23:35:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T23:35:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2] 2024-11-26T23:35:12,215 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:12,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:12,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:12,220 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:12,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:35:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:12,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/0e0c08b555c3420b81c8e0ee9d64feb4 is 50, key is test_row_0/A:col10/1732664112214/Put/seqid=0 2024-11-26T23:35:12,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742238_1414 (size=12001) 2024-11-26T23:35:12,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664172252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664172255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664172253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664172256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664172262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T23:35:12,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T23:35:12,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:12,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664172372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664172372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664172373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664172373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664172373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T23:35:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T23:35:12,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664172581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664172581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664172581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664172581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664172592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/0e0c08b555c3420b81c8e0ee9d64feb4 2024-11-26T23:35:12,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/1101fc55d1434b82a5dce9ad1ab4d733 is 50, key is test_row_0/B:col10/1732664112214/Put/seqid=0 2024-11-26T23:35:12,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T23:35:12,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:12,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742239_1415 (size=12001) 2024-11-26T23:35:12,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/1101fc55d1434b82a5dce9ad1ab4d733 2024-11-26T23:35:12,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/401586fe66f84fd896cb94dc65ccedb1 is 50, key is test_row_0/C:col10/1732664112214/Put/seqid=0 2024-11-26T23:35:12,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742240_1416 (size=12001) 2024-11-26T23:35:12,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T23:35:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T23:35:12,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664172891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664172891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664172891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664172891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664172912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:12,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T23:35:12,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:12,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:12,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:13,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/401586fe66f84fd896cb94dc65ccedb1 2024-11-26T23:35:13,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/0e0c08b555c3420b81c8e0ee9d64feb4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/0e0c08b555c3420b81c8e0ee9d64feb4 2024-11-26T23:35:13,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/0e0c08b555c3420b81c8e0ee9d64feb4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:35:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/1101fc55d1434b82a5dce9ad1ab4d733 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/1101fc55d1434b82a5dce9ad1ab4d733 2024-11-26T23:35:13,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/1101fc55d1434b82a5dce9ad1ab4d733, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:35:13,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/401586fe66f84fd896cb94dc65ccedb1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/401586fe66f84fd896cb94dc65ccedb1 2024-11-26T23:35:13,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/401586fe66f84fd896cb94dc65ccedb1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T23:35:13,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 88d8390f4cce6ec80971092bc0ed5e72 in 912ms, sequenceid=13, compaction requested=false 2024-11-26T23:35:13,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:13,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T23:35:13,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:13,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:35:13,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/066b5004788f470ab457c5da9b7e070f is 50, key is test_row_0/A:col10/1732664112252/Put/seqid=0 2024-11-26T23:35:13,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742241_1417 (size=12001) 2024-11-26T23:35:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T23:35:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664173422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664173422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664173422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664173423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664173430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/066b5004788f470ab457c5da9b7e070f 2024-11-26T23:35:13,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664173546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664173546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664173547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664173547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0488102ecd18430695e0320f22fa4a7a is 50, key is test_row_0/B:col10/1732664112252/Put/seqid=0 2024-11-26T23:35:13,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742242_1418 (size=12001) 2024-11-26T23:35:13,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664173761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664173761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664173761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664173761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:13,974 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0488102ecd18430695e0320f22fa4a7a 2024-11-26T23:35:13,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/568debd8eddf4c499bceb7fb23d30aea is 50, key is test_row_0/C:col10/1732664112252/Put/seqid=0 2024-11-26T23:35:14,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742243_1419 (size=12001) 2024-11-26T23:35:14,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/568debd8eddf4c499bceb7fb23d30aea 2024-11-26T23:35:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/066b5004788f470ab457c5da9b7e070f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/066b5004788f470ab457c5da9b7e070f 2024-11-26T23:35:14,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/066b5004788f470ab457c5da9b7e070f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T23:35:14,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0488102ecd18430695e0320f22fa4a7a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0488102ecd18430695e0320f22fa4a7a 2024-11-26T23:35:14,071 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0488102ecd18430695e0320f22fa4a7a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T23:35:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/568debd8eddf4c499bceb7fb23d30aea as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/568debd8eddf4c499bceb7fb23d30aea 2024-11-26T23:35:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664174071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664174071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664174072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664174072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,081 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/568debd8eddf4c499bceb7fb23d30aea, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T23:35:14,082 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 88d8390f4cce6ec80971092bc0ed5e72 in 947ms, sequenceid=37, compaction requested=false 2024-11-26T23:35:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-26T23:35:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-26T23:35:14,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-26T23:35:14,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-26T23:35:14,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8850 sec 2024-11-26T23:35:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T23:35:14,362 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:14,364 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-26T23:35:14,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T23:35:14,367 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:14,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:14,446 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:14,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T23:35:14,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/249b1b3aaf39428d801e533fffaf95eb is 50, key is test_row_0/A:col10/1732664113428/Put/seqid=0 2024-11-26T23:35:14,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742244_1420 (size=12001) 2024-11-26T23:35:14,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/249b1b3aaf39428d801e533fffaf95eb 2024-11-26T23:35:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T23:35:14,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/02207980f5324d9b8909c91418cf68cd is 50, key is test_row_0/B:col10/1732664113428/Put/seqid=0 2024-11-26T23:35:14,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742245_1421 (size=12001) 2024-11-26T23:35:14,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/02207980f5324d9b8909c91418cf68cd 2024-11-26T23:35:14,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/238ea3a27e624d5db0fd7b3b6ee42a2f is 50, key is test_row_0/C:col10/1732664113428/Put/seqid=0 2024-11-26T23:35:14,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742246_1422 (size=12001) 2024-11-26T23:35:14,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T23:35:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664174600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664174601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664174612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664174616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664174616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T23:35:14,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T23:35:14,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664174721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664174721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664174732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664174732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664174733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T23:35:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/238ea3a27e624d5db0fd7b3b6ee42a2f 2024-11-26T23:35:14,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/249b1b3aaf39428d801e533fffaf95eb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/249b1b3aaf39428d801e533fffaf95eb 2024-11-26T23:35:14,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/249b1b3aaf39428d801e533fffaf95eb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T23:35:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/02207980f5324d9b8909c91418cf68cd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/02207980f5324d9b8909c91418cf68cd 2024-11-26T23:35:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664174943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664174944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:14,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/02207980f5324d9b8909c91418cf68cd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T23:35:14,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/238ea3a27e624d5db0fd7b3b6ee42a2f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/238ea3a27e624d5db0fd7b3b6ee42a2f 2024-11-26T23:35:14,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/238ea3a27e624d5db0fd7b3b6ee42a2f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T23:35:14,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 88d8390f4cce6ec80971092bc0ed5e72 in 511ms, sequenceid=50, compaction requested=true 2024-11-26T23:35:14,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:14,958 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:14,961 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:14,961 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:14,961 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,961 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/0e0c08b555c3420b81c8e0ee9d64feb4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/066b5004788f470ab457c5da9b7e070f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/249b1b3aaf39428d801e533fffaf95eb] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=35.2 K 2024-11-26T23:35:14,962 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e0c08b555c3420b81c8e0ee9d64feb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732664112214 2024-11-26T23:35:14,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:14,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:14,962 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:14,962 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 066b5004788f470ab457c5da9b7e070f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732664112252 2024-11-26T23:35:14,963 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249b1b3aaf39428d801e533fffaf95eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732664113422 2024-11-26T23:35:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:14,963 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:14,963 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:14,963 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,963 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/1101fc55d1434b82a5dce9ad1ab4d733, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0488102ecd18430695e0320f22fa4a7a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/02207980f5324d9b8909c91418cf68cd] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=35.2 K 2024-11-26T23:35:14,964 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 1101fc55d1434b82a5dce9ad1ab4d733, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732664112214 2024-11-26T23:35:14,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:14,964 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0488102ecd18430695e0320f22fa4a7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732664112252 2024-11-26T23:35:14,965 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 02207980f5324d9b8909c91418cf68cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732664113422 2024-11-26T23:35:14,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:14,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:14,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:14,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:14,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T23:35:14,981 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:14,981 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/b1fb9740eabc4ae9b4adfa2b3ef5958c is 50, key is test_row_0/A:col10/1732664113428/Put/seqid=0 2024-11-26T23:35:14,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T23:35:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:14,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T23:35:14,999 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:14,999 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/950548fbc35f42b68115ef1d397bb876 is 50, key is test_row_0/B:col10/1732664113428/Put/seqid=0 2024-11-26T23:35:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664174997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664174997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664174999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/c59c49bfd9c1441f953ae00abed18872 is 50, key is test_row_0/A:col10/1732664114965/Put/seqid=0 2024-11-26T23:35:15,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742247_1423 (size=12104) 2024-11-26T23:35:15,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742248_1424 (size=12104) 2024-11-26T23:35:15,049 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/950548fbc35f42b68115ef1d397bb876 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/950548fbc35f42b68115ef1d397bb876 2024-11-26T23:35:15,049 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/b1fb9740eabc4ae9b4adfa2b3ef5958c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b1fb9740eabc4ae9b4adfa2b3ef5958c 2024-11-26T23:35:15,056 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 950548fbc35f42b68115ef1d397bb876(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:15,056 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:15,056 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=13, startTime=1732664114962; duration=0sec 2024-11-26T23:35:15,057 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:15,057 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:15,057 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:15,057 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:15,057 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:15,058 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:15,058 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/401586fe66f84fd896cb94dc65ccedb1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/568debd8eddf4c499bceb7fb23d30aea, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/238ea3a27e624d5db0fd7b3b6ee42a2f] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=35.2 K 2024-11-26T23:35:15,058 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 401586fe66f84fd896cb94dc65ccedb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732664112214 2024-11-26T23:35:15,058 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 568debd8eddf4c499bceb7fb23d30aea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732664112252 2024-11-26T23:35:15,059 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 238ea3a27e624d5db0fd7b3b6ee42a2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732664113422 2024-11-26T23:35:15,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742249_1425 (size=12001) 2024-11-26T23:35:15,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/c59c49bfd9c1441f953ae00abed18872 2024-11-26T23:35:15,094 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into b1fb9740eabc4ae9b4adfa2b3ef5958c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:15,094 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:15,094 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=13, startTime=1732664114957; duration=0sec 2024-11-26T23:35:15,094 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:15,094 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:15,099 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:35:15,100 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:15,101 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/fc2be232fdbf48bd96cbe05df17ad6d2 is 50, key is test_row_0/C:col10/1732664113428/Put/seqid=0 2024-11-26T23:35:15,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/4e9c51501709428086c3f18c240f72ac is 50, key is test_row_0/B:col10/1732664114965/Put/seqid=0 2024-11-26T23:35:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664175111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664175111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664175112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T23:35:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:15,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:15,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742250_1426 (size=12104) 2024-11-26T23:35:15,176 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/fc2be232fdbf48bd96cbe05df17ad6d2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fc2be232fdbf48bd96cbe05df17ad6d2 2024-11-26T23:35:15,180 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into fc2be232fdbf48bd96cbe05df17ad6d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:15,180 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:15,180 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=13, startTime=1732664114967; duration=0sec 2024-11-26T23:35:15,181 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:15,181 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:15,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742251_1427 (size=12001) 2024-11-26T23:35:15,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/4e9c51501709428086c3f18c240f72ac 2024-11-26T23:35:15,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/f1ca4a93c37a4271860611d76b0e0c1f is 50, key is test_row_0/C:col10/1732664114965/Put/seqid=0 2024-11-26T23:35:15,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742252_1428 (size=12001) 2024-11-26T23:35:15,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/f1ca4a93c37a4271860611d76b0e0c1f 2024-11-26T23:35:15,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664175252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664175252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/c59c49bfd9c1441f953ae00abed18872 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c59c49bfd9c1441f953ae00abed18872 2024-11-26T23:35:15,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c59c49bfd9c1441f953ae00abed18872, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T23:35:15,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/4e9c51501709428086c3f18c240f72ac as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/4e9c51501709428086c3f18c240f72ac 2024-11-26T23:35:15,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/4e9c51501709428086c3f18c240f72ac, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T23:35:15,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/f1ca4a93c37a4271860611d76b0e0c1f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f1ca4a93c37a4271860611d76b0e0c1f 2024-11-26T23:35:15,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f1ca4a93c37a4271860611d76b0e0c1f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T23:35:15,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 88d8390f4cce6ec80971092bc0ed5e72 in 297ms, sequenceid=76, compaction requested=false 2024-11-26T23:35:15,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:15,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T23:35:15,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:15,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T23:35:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/a9f659c8fcd8464d824506f7b0b12463 is 50, key is test_row_0/A:col10/1732664114978/Put/seqid=0 2024-11-26T23:35:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:15,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742253_1429 (size=12001) 2024-11-26T23:35:15,340 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/a9f659c8fcd8464d824506f7b0b12463 2024-11-26T23:35:15,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/732bd1b523da4248a59409bc46ef665d is 50, key is test_row_0/B:col10/1732664114978/Put/seqid=0 2024-11-26T23:35:15,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742254_1430 (size=12001) 2024-11-26T23:35:15,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664175438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664175449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664175451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T23:35:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664175562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664175562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664175572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,755 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/732bd1b523da4248a59409bc46ef665d 2024-11-26T23:35:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664175763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664175763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/51d19e93a1554a4598a9cf5dcd3541a0 is 50, key is test_row_0/C:col10/1732664114978/Put/seqid=0 2024-11-26T23:35:15,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664175772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664175772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664175792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:15,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742255_1431 (size=12001) 2024-11-26T23:35:16,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664176092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664176092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664176111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/51d19e93a1554a4598a9cf5dcd3541a0 2024-11-26T23:35:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/a9f659c8fcd8464d824506f7b0b12463 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/a9f659c8fcd8464d824506f7b0b12463 2024-11-26T23:35:16,219 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/a9f659c8fcd8464d824506f7b0b12463, entries=150, sequenceid=89, filesize=11.7 K 2024-11-26T23:35:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/732bd1b523da4248a59409bc46ef665d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/732bd1b523da4248a59409bc46ef665d 2024-11-26T23:35:16,223 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/732bd1b523da4248a59409bc46ef665d, entries=150, sequenceid=89, filesize=11.7 K 2024-11-26T23:35:16,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/51d19e93a1554a4598a9cf5dcd3541a0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/51d19e93a1554a4598a9cf5dcd3541a0 2024-11-26T23:35:16,227 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/51d19e93a1554a4598a9cf5dcd3541a0, entries=150, sequenceid=89, filesize=11.7 K 2024-11-26T23:35:16,228 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 88d8390f4cce6ec80971092bc0ed5e72 in 937ms, sequenceid=89, compaction requested=true 2024-11-26T23:35:16,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:16,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:16,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-26T23:35:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-26T23:35:16,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-26T23:35:16,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-11-26T23:35:16,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8680 sec 2024-11-26T23:35:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T23:35:16,512 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:16,512 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-26T23:35:16,514 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T23:35:16,515 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:16,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:16,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:16,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-26T23:35:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T23:35:16,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/170dd17ad3c44a4aaeb982ab883291dc is 50, key is test_row_0/A:col10/1732664115448/Put/seqid=0 2024-11-26T23:35:16,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742256_1432 (size=14341) 2024-11-26T23:35:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664176636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664176640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664176642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:16,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:16,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:16,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:16,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664176751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664176751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664176762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664176782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664176782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,790 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 88d8390f4cce6ec80971092bc0ed5e72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,790 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 88d8390f4cce6ec80971092bc0ed5e72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:16,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T23:35:16,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664176962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664176962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:16,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:16,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:16,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:16,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664176981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:17,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/170dd17ad3c44a4aaeb982ab883291dc 2024-11-26T23:35:17,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/dfb66b33bd9e4462ad9d653e2314db85 is 50, key is test_row_0/B:col10/1732664115448/Put/seqid=0 2024-11-26T23:35:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742257_1433 (size=12001) 2024-11-26T23:35:17,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:17,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:17,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T23:35:17,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664177282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:17,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664177283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:17,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664177301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:17,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/dfb66b33bd9e4462ad9d653e2314db85 2024-11-26T23:35:17,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/bad827cfdb23458c95bcb21085c78fff is 50, key is test_row_0/C:col10/1732664115448/Put/seqid=0 2024-11-26T23:35:17,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742258_1434 (size=12001) 2024-11-26T23:35:17,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T23:35:17,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664177791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:17,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664177791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:17,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664177822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:17,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/bad827cfdb23458c95bcb21085c78fff 2024-11-26T23:35:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/170dd17ad3c44a4aaeb982ab883291dc as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/170dd17ad3c44a4aaeb982ab883291dc 2024-11-26T23:35:17,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/170dd17ad3c44a4aaeb982ab883291dc, entries=200, sequenceid=117, filesize=14.0 K 2024-11-26T23:35:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/dfb66b33bd9e4462ad9d653e2314db85 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/dfb66b33bd9e4462ad9d653e2314db85 2024-11-26T23:35:17,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/dfb66b33bd9e4462ad9d653e2314db85, entries=150, sequenceid=117, filesize=11.7 K 2024-11-26T23:35:17,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:17,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/bad827cfdb23458c95bcb21085c78fff as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/bad827cfdb23458c95bcb21085c78fff 2024-11-26T23:35:17,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:17,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:17,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/bad827cfdb23458c95bcb21085c78fff, entries=150, sequenceid=117, filesize=11.7 K 2024-11-26T23:35:17,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 88d8390f4cce6ec80971092bc0ed5e72 in 1279ms, sequenceid=117, compaction requested=true 2024-11-26T23:35:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:17,901 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:17,901 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:17,903 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:17,903 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:17,903 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,903 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b1fb9740eabc4ae9b4adfa2b3ef5958c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c59c49bfd9c1441f953ae00abed18872, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/a9f659c8fcd8464d824506f7b0b12463, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/170dd17ad3c44a4aaeb982ab883291dc] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=49.3 K 2024-11-26T23:35:17,903 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:17,903 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:17,903 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:17,903 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/950548fbc35f42b68115ef1d397bb876, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/4e9c51501709428086c3f18c240f72ac, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/732bd1b523da4248a59409bc46ef665d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/dfb66b33bd9e4462ad9d653e2314db85] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=47.0 K 2024-11-26T23:35:17,904 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1fb9740eabc4ae9b4adfa2b3ef5958c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732664113422 2024-11-26T23:35:17,904 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting c59c49bfd9c1441f953ae00abed18872, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732664114612 2024-11-26T23:35:17,904 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 950548fbc35f42b68115ef1d397bb876, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732664113422 2024-11-26T23:35:17,912 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f659c8fcd8464d824506f7b0b12463, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732664114978 2024-11-26T23:35:17,913 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9c51501709428086c3f18c240f72ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732664114612 2024-11-26T23:35:17,913 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 170dd17ad3c44a4aaeb982ab883291dc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732664115433 2024-11-26T23:35:17,913 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 732bd1b523da4248a59409bc46ef665d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732664114978 2024-11-26T23:35:17,916 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb66b33bd9e4462ad9d653e2314db85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732664115447 2024-11-26T23:35:17,927 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:17,928 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/c61b76f81cb34f6799a0aafe5d8ecb24 is 50, key is test_row_0/A:col10/1732664115448/Put/seqid=0 2024-11-26T23:35:17,934 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:17,934 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/f263f074217d47b79fcfe5f186e653c6 is 50, key is test_row_0/B:col10/1732664115448/Put/seqid=0 2024-11-26T23:35:17,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742259_1435 (size=12241) 2024-11-26T23:35:17,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742260_1436 (size=12241) 2024-11-26T23:35:18,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T23:35:18,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:18,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-26T23:35:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/61ed81c660c0497cb634b78ab0b68731 is 50, key is test_row_0/A:col10/1732664116629/Put/seqid=0 2024-11-26T23:35:18,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742261_1437 (size=12001) 2024-11-26T23:35:18,112 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/61ed81c660c0497cb634b78ab0b68731 2024-11-26T23:35:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6c7eedab78a04c7292432942c569b78b is 50, key is test_row_0/B:col10/1732664116629/Put/seqid=0 2024-11-26T23:35:18,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742262_1438 (size=12001) 2024-11-26T23:35:18,354 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/c61b76f81cb34f6799a0aafe5d8ecb24 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c61b76f81cb34f6799a0aafe5d8ecb24 2024-11-26T23:35:18,361 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into c61b76f81cb34f6799a0aafe5d8ecb24(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:18,361 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:18,361 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=12, startTime=1732664117901; duration=0sec 2024-11-26T23:35:18,361 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:18,361 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:18,361 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:18,366 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:18,366 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:18,366 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:18,366 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fc2be232fdbf48bd96cbe05df17ad6d2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f1ca4a93c37a4271860611d76b0e0c1f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/51d19e93a1554a4598a9cf5dcd3541a0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/bad827cfdb23458c95bcb21085c78fff] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=47.0 K 2024-11-26T23:35:18,367 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/f263f074217d47b79fcfe5f186e653c6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/f263f074217d47b79fcfe5f186e653c6 2024-11-26T23:35:18,370 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc2be232fdbf48bd96cbe05df17ad6d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732664113422 2024-11-26T23:35:18,373 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into f263f074217d47b79fcfe5f186e653c6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:18,373 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:18,374 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=12, startTime=1732664117901; duration=0sec 2024-11-26T23:35:18,374 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:18,374 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:18,374 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ca4a93c37a4271860611d76b0e0c1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732664114612 2024-11-26T23:35:18,374 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d19e93a1554a4598a9cf5dcd3541a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732664114978 2024-11-26T23:35:18,374 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad827cfdb23458c95bcb21085c78fff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732664115447 2024-11-26T23:35:18,382 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:18,382 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b556a69fa9d24d82b9902c46080b6017 is 50, key is test_row_0/C:col10/1732664115448/Put/seqid=0 2024-11-26T23:35:18,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742263_1439 (size=12241) 2024-11-26T23:35:18,445 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b556a69fa9d24d82b9902c46080b6017 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b556a69fa9d24d82b9902c46080b6017 2024-11-26T23:35:18,457 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into b556a69fa9d24d82b9902c46080b6017(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:18,457 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:18,457 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=12, startTime=1732664117902; duration=0sec 2024-11-26T23:35:18,458 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:18,458 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:18,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6c7eedab78a04c7292432942c569b78b 2024-11-26T23:35:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/2bd8b710b15d41e9bb100f4a9deaf56c is 50, key is test_row_0/C:col10/1732664116629/Put/seqid=0 2024-11-26T23:35:18,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742264_1440 (size=12001) 2024-11-26T23:35:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T23:35:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:18,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:18,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664178855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:18,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664178861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:18,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664178864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664178869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:18,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664178870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:18,958 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/2bd8b710b15d41e9bb100f4a9deaf56c 2024-11-26T23:35:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/61ed81c660c0497cb634b78ab0b68731 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/61ed81c660c0497cb634b78ab0b68731 2024-11-26T23:35:18,967 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/61ed81c660c0497cb634b78ab0b68731, entries=150, sequenceid=125, filesize=11.7 K 2024-11-26T23:35:18,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6c7eedab78a04c7292432942c569b78b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6c7eedab78a04c7292432942c569b78b 2024-11-26T23:35:18,972 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6c7eedab78a04c7292432942c569b78b, entries=150, sequenceid=125, filesize=11.7 K 2024-11-26T23:35:18,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/2bd8b710b15d41e9bb100f4a9deaf56c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2bd8b710b15d41e9bb100f4a9deaf56c 2024-11-26T23:35:18,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2bd8b710b15d41e9bb100f4a9deaf56c, entries=150, sequenceid=125, filesize=11.7 K 2024-11-26T23:35:18,977 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 88d8390f4cce6ec80971092bc0ed5e72 in 929ms, sequenceid=125, compaction requested=false 2024-11-26T23:35:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-26T23:35:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-26T23:35:18,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-26T23:35:18,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4650 sec 2024-11-26T23:35:19,001 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:19,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-26T23:35:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:19,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.4700 sec 2024-11-26T23:35:19,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664179001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664179002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7d11d24e16204385a9308584595c7f8d is 50, key is test_row_0/A:col10/1732664118997/Put/seqid=0 2024-11-26T23:35:19,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664179004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664179004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664179006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742265_1441 (size=14541) 2024-11-26T23:35:19,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7d11d24e16204385a9308584595c7f8d 2024-11-26T23:35:19,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/3fb6eaaaa9fe4d9ca668b7b44aa70775 is 50, key is test_row_0/B:col10/1732664118997/Put/seqid=0 2024-11-26T23:35:19,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742266_1442 (size=12151) 2024-11-26T23:35:19,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/3fb6eaaaa9fe4d9ca668b7b44aa70775 2024-11-26T23:35:19,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b99e5b80971045bd84020bd216cc4671 is 50, key is test_row_0/C:col10/1732664118997/Put/seqid=0 2024-11-26T23:35:19,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742267_1443 (size=12151) 2024-11-26T23:35:19,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b99e5b80971045bd84020bd216cc4671 2024-11-26T23:35:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7d11d24e16204385a9308584595c7f8d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7d11d24e16204385a9308584595c7f8d 2024-11-26T23:35:19,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7d11d24e16204385a9308584595c7f8d, entries=200, sequenceid=160, filesize=14.2 K 2024-11-26T23:35:19,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/3fb6eaaaa9fe4d9ca668b7b44aa70775 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/3fb6eaaaa9fe4d9ca668b7b44aa70775 2024-11-26T23:35:19,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/3fb6eaaaa9fe4d9ca668b7b44aa70775, entries=150, sequenceid=160, filesize=11.9 K 2024-11-26T23:35:19,109 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-26T23:35:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b99e5b80971045bd84020bd216cc4671 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b99e5b80971045bd84020bd216cc4671 2024-11-26T23:35:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664179111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664179111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b99e5b80971045bd84020bd216cc4671, entries=150, sequenceid=160, filesize=11.9 K 2024-11-26T23:35:19,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 88d8390f4cce6ec80971092bc0ed5e72 in 118ms, sequenceid=160, compaction requested=true 2024-11-26T23:35:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:19,120 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:19,120 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:19,128 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:19,128 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:19,128 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:19,128 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c61b76f81cb34f6799a0aafe5d8ecb24, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/61ed81c660c0497cb634b78ab0b68731, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7d11d24e16204385a9308584595c7f8d] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=37.9 K 2024-11-26T23:35:19,129 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:19,129 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:19,129 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:19,129 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/f263f074217d47b79fcfe5f186e653c6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6c7eedab78a04c7292432942c569b78b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/3fb6eaaaa9fe4d9ca668b7b44aa70775] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=35.5 K 2024-11-26T23:35:19,129 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting c61b76f81cb34f6799a0aafe5d8ecb24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732664115447 2024-11-26T23:35:19,130 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f263f074217d47b79fcfe5f186e653c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732664115447 2024-11-26T23:35:19,130 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ed81c660c0497cb634b78ab0b68731, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732664116629 2024-11-26T23:35:19,130 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c7eedab78a04c7292432942c569b78b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732664116629 2024-11-26T23:35:19,130 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb6eaaaa9fe4d9ca668b7b44aa70775, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664118996 2024-11-26T23:35:19,130 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d11d24e16204385a9308584595c7f8d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664118996 2024-11-26T23:35:19,137 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:19,137 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:19,138 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/d50d9a80539c47c6b5cf111445770848 is 50, key is test_row_0/A:col10/1732664118997/Put/seqid=0 2024-11-26T23:35:19,138 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0cec1ef383da4366b07f8458982b36e8 is 50, key is test_row_0/B:col10/1732664118997/Put/seqid=0 2024-11-26T23:35:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:19,145 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:19,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:35:19,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742268_1444 (size=12493) 2024-11-26T23:35:19,164 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/d50d9a80539c47c6b5cf111445770848 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d50d9a80539c47c6b5cf111445770848 2024-11-26T23:35:19,170 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into d50d9a80539c47c6b5cf111445770848(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:19,170 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:19,170 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=13, startTime=1732664119119; duration=0sec 2024-11-26T23:35:19,170 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:19,170 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:19,170 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:19,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742269_1445 (size=12493) 2024-11-26T23:35:19,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/fb1414a06f724bd39dd198a129e7b260 is 50, key is test_row_0/A:col10/1732664119136/Put/seqid=0 2024-11-26T23:35:19,177 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0cec1ef383da4366b07f8458982b36e8 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0cec1ef383da4366b07f8458982b36e8 2024-11-26T23:35:19,180 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 0cec1ef383da4366b07f8458982b36e8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:19,180 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:19,180 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=13, startTime=1732664119120; duration=0sec 2024-11-26T23:35:19,180 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:19,180 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:19,180 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:19,180 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:19,180 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:19,181 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b556a69fa9d24d82b9902c46080b6017, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2bd8b710b15d41e9bb100f4a9deaf56c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b99e5b80971045bd84020bd216cc4671] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=35.5 K 2024-11-26T23:35:19,181 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b556a69fa9d24d82b9902c46080b6017, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732664115447 2024-11-26T23:35:19,181 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd8b710b15d41e9bb100f4a9deaf56c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732664116629 2024-11-26T23:35:19,182 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99e5b80971045bd84020bd216cc4671, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664118996 2024-11-26T23:35:19,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742270_1446 (size=14541) 2024-11-26T23:35:19,203 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:19,203 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/55aad944b1e649ad9a31ebf9bf6a2129 is 50, key is test_row_0/C:col10/1732664118997/Put/seqid=0 2024-11-26T23:35:19,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/fb1414a06f724bd39dd198a129e7b260 2024-11-26T23:35:19,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e0c5ec37ecee4db3b3702fad51a8b401 is 50, key is test_row_0/B:col10/1732664119136/Put/seqid=0 2024-11-26T23:35:19,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742271_1447 (size=12493) 2024-11-26T23:35:19,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742272_1448 (size=12151) 2024-11-26T23:35:19,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e0c5ec37ecee4db3b3702fad51a8b401 2024-11-26T23:35:19,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b0e9641399d0470d80819110ae965a69 is 50, key is test_row_0/C:col10/1732664119136/Put/seqid=0 2024-11-26T23:35:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664179243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664179245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742273_1449 (size=12151) 2024-11-26T23:35:19,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b0e9641399d0470d80819110ae965a69 2024-11-26T23:35:19,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/fb1414a06f724bd39dd198a129e7b260 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/fb1414a06f724bd39dd198a129e7b260 2024-11-26T23:35:19,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664179258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/fb1414a06f724bd39dd198a129e7b260, entries=200, sequenceid=172, filesize=14.2 K 2024-11-26T23:35:19,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e0c5ec37ecee4db3b3702fad51a8b401 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e0c5ec37ecee4db3b3702fad51a8b401 2024-11-26T23:35:19,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e0c5ec37ecee4db3b3702fad51a8b401, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T23:35:19,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b0e9641399d0470d80819110ae965a69 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b0e9641399d0470d80819110ae965a69 2024-11-26T23:35:19,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b0e9641399d0470d80819110ae965a69, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T23:35:19,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 88d8390f4cce6ec80971092bc0ed5e72 in 136ms, sequenceid=172, compaction requested=false 2024-11-26T23:35:19,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:19,323 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:19,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:35:19,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7ebd55fc00f84d4c8ebdbf100474d177 is 50, key is test_row_0/A:col10/1732664119223/Put/seqid=0 2024-11-26T23:35:19,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742274_1450 (size=14541) 2024-11-26T23:35:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664179351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664179353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664179362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664179362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664179381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664179472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664179472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664179582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664179582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664179591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,651 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/55aad944b1e649ad9a31ebf9bf6a2129 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/55aad944b1e649ad9a31ebf9bf6a2129 2024-11-26T23:35:19,658 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into 55aad944b1e649ad9a31ebf9bf6a2129(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:19,658 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:19,658 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=13, startTime=1732664119120; duration=0sec 2024-11-26T23:35:19,659 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:19,659 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:19,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664179691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664179691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7ebd55fc00f84d4c8ebdbf100474d177 2024-11-26T23:35:19,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/d07a2d5356184510a40c4c04add3ad24 is 50, key is test_row_0/B:col10/1732664119223/Put/seqid=0 2024-11-26T23:35:19,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742275_1451 (size=12151) 2024-11-26T23:35:19,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/d07a2d5356184510a40c4c04add3ad24 2024-11-26T23:35:19,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/fe81a2350f294b2ba0b094e094409b2d is 50, key is test_row_0/C:col10/1732664119223/Put/seqid=0 2024-11-26T23:35:19,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742276_1452 (size=12151) 2024-11-26T23:35:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664179892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664179892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:19,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664179901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664180011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664180011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/fe81a2350f294b2ba0b094e094409b2d 2024-11-26T23:35:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7ebd55fc00f84d4c8ebdbf100474d177 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7ebd55fc00f84d4c8ebdbf100474d177 2024-11-26T23:35:20,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7ebd55fc00f84d4c8ebdbf100474d177, entries=200, sequenceid=199, filesize=14.2 K 2024-11-26T23:35:20,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/d07a2d5356184510a40c4c04add3ad24 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/d07a2d5356184510a40c4c04add3ad24 2024-11-26T23:35:20,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/d07a2d5356184510a40c4c04add3ad24, entries=150, sequenceid=199, filesize=11.9 K 2024-11-26T23:35:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/fe81a2350f294b2ba0b094e094409b2d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fe81a2350f294b2ba0b094e094409b2d 2024-11-26T23:35:20,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fe81a2350f294b2ba0b094e094409b2d, entries=150, sequenceid=199, filesize=11.9 K 2024-11-26T23:35:20,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 88d8390f4cce6ec80971092bc0ed5e72 in 863ms, sequenceid=199, compaction requested=true 2024-11-26T23:35:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:20,186 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:20,187 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:20,187 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:20,187 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:20,187 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d50d9a80539c47c6b5cf111445770848, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/fb1414a06f724bd39dd198a129e7b260, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7ebd55fc00f84d4c8ebdbf100474d177] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=40.6 K 2024-11-26T23:35:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:20,187 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:20,187 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d50d9a80539c47c6b5cf111445770848, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664118996 2024-11-26T23:35:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:20,189 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb1414a06f724bd39dd198a129e7b260, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664119128 2024-11-26T23:35:20,189 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:20,189 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:20,189 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:20,189 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0cec1ef383da4366b07f8458982b36e8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e0c5ec37ecee4db3b3702fad51a8b401, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/d07a2d5356184510a40c4c04add3ad24] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=35.9 K 2024-11-26T23:35:20,189 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cec1ef383da4366b07f8458982b36e8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664118996 2024-11-26T23:35:20,189 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ebd55fc00f84d4c8ebdbf100474d177, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664119223 2024-11-26T23:35:20,190 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e0c5ec37ecee4db3b3702fad51a8b401, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664119128 2024-11-26T23:35:20,192 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d07a2d5356184510a40c4c04add3ad24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664119223 2024-11-26T23:35:20,203 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:20,204 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:20,204 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/67798ab9617e45479627901a0f70d0d2 is 50, key is test_row_0/B:col10/1732664119223/Put/seqid=0 2024-11-26T23:35:20,204 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/743d1d6a7a5e4da7a908a6fa73831e05 is 50, key is test_row_0/A:col10/1732664119223/Put/seqid=0 2024-11-26T23:35:20,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742278_1454 (size=12595) 2024-11-26T23:35:20,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742277_1453 (size=12595) 2024-11-26T23:35:20,416 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:20,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T23:35:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:20,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/043b67c9e0e848c2914c0a3f10101de7 is 50, key is test_row_0/A:col10/1732664120415/Put/seqid=0 2024-11-26T23:35:20,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742279_1455 (size=14541) 2024-11-26T23:35:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664180525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664180525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664180529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664180532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664180532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,627 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/67798ab9617e45479627901a0f70d0d2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/67798ab9617e45479627901a0f70d0d2 2024-11-26T23:35:20,638 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/743d1d6a7a5e4da7a908a6fa73831e05 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/743d1d6a7a5e4da7a908a6fa73831e05 2024-11-26T23:35:20,640 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 67798ab9617e45479627901a0f70d0d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:20,640 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:20,640 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=13, startTime=1732664120187; duration=0sec 2024-11-26T23:35:20,640 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:20,640 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:20,640 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:20,641 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:20,641 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:20,641 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:20,641 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/55aad944b1e649ad9a31ebf9bf6a2129, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b0e9641399d0470d80819110ae965a69, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fe81a2350f294b2ba0b094e094409b2d] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=35.9 K 2024-11-26T23:35:20,641 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 55aad944b1e649ad9a31ebf9bf6a2129, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664118996 2024-11-26T23:35:20,642 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e9641399d0470d80819110ae965a69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732664119128 2024-11-26T23:35:20,642 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting fe81a2350f294b2ba0b094e094409b2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664119223 2024-11-26T23:35:20,644 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into 743d1d6a7a5e4da7a908a6fa73831e05(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:20,644 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:20,644 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=13, startTime=1732664120186; duration=0sec 2024-11-26T23:35:20,644 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:20,644 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664180642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664180642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,662 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:20,663 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/4aced4302a1e43d2bfbf16585bb989ef is 50, key is test_row_0/C:col10/1732664119223/Put/seqid=0 2024-11-26T23:35:20,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664180651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T23:35:20,672 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:20,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-26T23:35:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T23:35:20,675 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:20,676 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:20,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:20,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742280_1456 (size=12595) 2024-11-26T23:35:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T23:35:20,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T23:35:20,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:20,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:20,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:20,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:20,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:20,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/043b67c9e0e848c2914c0a3f10101de7 2024-11-26T23:35:20,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/b4a5bf45068b4b619a7dc59e12abda82 is 50, key is test_row_0/B:col10/1732664120415/Put/seqid=0 2024-11-26T23:35:20,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742281_1457 (size=12151) 2024-11-26T23:35:20,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/b4a5bf45068b4b619a7dc59e12abda82 2024-11-26T23:35:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664180863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664180864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664180871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:20,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/d0e3c0b21eb84a308cd25e6fe437561a is 50, key is test_row_0/C:col10/1732664120415/Put/seqid=0 2024-11-26T23:35:20,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742282_1458 (size=12151) 2024-11-26T23:35:20,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/d0e3c0b21eb84a308cd25e6fe437561a 2024-11-26T23:35:20,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/043b67c9e0e848c2914c0a3f10101de7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/043b67c9e0e848c2914c0a3f10101de7 2024-11-26T23:35:20,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/043b67c9e0e848c2914c0a3f10101de7, entries=200, sequenceid=212, filesize=14.2 K 2024-11-26T23:35:20,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/b4a5bf45068b4b619a7dc59e12abda82 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b4a5bf45068b4b619a7dc59e12abda82 2024-11-26T23:35:20,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b4a5bf45068b4b619a7dc59e12abda82, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T23:35:20,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/d0e3c0b21eb84a308cd25e6fe437561a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/d0e3c0b21eb84a308cd25e6fe437561a 2024-11-26T23:35:20,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/d0e3c0b21eb84a308cd25e6fe437561a, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T23:35:20,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 88d8390f4cce6ec80971092bc0ed5e72 in 516ms, sequenceid=212, compaction requested=false 2024-11-26T23:35:20,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:20,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T23:35:20,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:20,982 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:35:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/157bcc315e8e469cbea6aa7e24591d5d is 50, key is test_row_0/A:col10/1732664120520/Put/seqid=0 2024-11-26T23:35:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T23:35:20,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742283_1459 (size=12151) 2024-11-26T23:35:21,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/157bcc315e8e469cbea6aa7e24591d5d 2024-11-26T23:35:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e2744457d1a84c22bbc17551ddaa8709 is 50, key is test_row_0/B:col10/1732664120520/Put/seqid=0 2024-11-26T23:35:21,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742284_1460 (size=12151) 2024-11-26T23:35:21,134 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/4aced4302a1e43d2bfbf16585bb989ef as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/4aced4302a1e43d2bfbf16585bb989ef 2024-11-26T23:35:21,146 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into 4aced4302a1e43d2bfbf16585bb989ef(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:21,146 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:21,146 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=13, startTime=1732664120188; duration=0sec 2024-11-26T23:35:21,146 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:21,146 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:21,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:21,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664181214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664181215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664181220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T23:35:21,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664181332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664181332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664181341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e2744457d1a84c22bbc17551ddaa8709 2024-11-26T23:35:21,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/98e8037f92514a42a08db9f06978177e is 50, key is test_row_0/C:col10/1732664120520/Put/seqid=0 2024-11-26T23:35:21,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742285_1461 (size=12151) 2024-11-26T23:35:21,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/98e8037f92514a42a08db9f06978177e 2024-11-26T23:35:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/157bcc315e8e469cbea6aa7e24591d5d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/157bcc315e8e469cbea6aa7e24591d5d 2024-11-26T23:35:21,450 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/157bcc315e8e469cbea6aa7e24591d5d, entries=150, sequenceid=238, filesize=11.9 K 2024-11-26T23:35:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e2744457d1a84c22bbc17551ddaa8709 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e2744457d1a84c22bbc17551ddaa8709 2024-11-26T23:35:21,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e2744457d1a84c22bbc17551ddaa8709, entries=150, sequenceid=238, filesize=11.9 K 2024-11-26T23:35:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/98e8037f92514a42a08db9f06978177e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/98e8037f92514a42a08db9f06978177e 2024-11-26T23:35:21,457 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/98e8037f92514a42a08db9f06978177e, entries=150, sequenceid=238, filesize=11.9 K 2024-11-26T23:35:21,458 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 88d8390f4cce6ec80971092bc0ed5e72 in 476ms, sequenceid=238, compaction requested=true 2024-11-26T23:35:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-26T23:35:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-26T23:35:21,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-26T23:35:21,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 783 msec 2024-11-26T23:35:21,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 788 msec 2024-11-26T23:35:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:21,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:21,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T23:35:21,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/14c9ee6387714cb1a70f7b19dd389c54 is 50, key is test_row_0/A:col10/1732664121555/Put/seqid=0 2024-11-26T23:35:21,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742286_1462 (size=16931) 2024-11-26T23:35:21,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/14c9ee6387714cb1a70f7b19dd389c54 2024-11-26T23:35:21,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/5fcf46b740544b97aada09dc60abc28a is 50, key is test_row_0/B:col10/1732664121555/Put/seqid=0 2024-11-26T23:35:21,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742287_1463 (size=12151) 2024-11-26T23:35:21,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664181593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664181600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664181601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664181605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664181606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664181712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664181722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664181722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664181722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664181731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T23:35:21,812 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:21,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-26T23:35:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T23:35:21,814 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:21,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:21,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T23:35:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664181932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664181941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664181941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664181941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664181951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:21,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T23:35:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:21,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:21,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/5fcf46b740544b97aada09dc60abc28a 2024-11-26T23:35:21,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/8b92a36e5a8a49b1afea4c98fa2d4286 is 50, key is test_row_0/C:col10/1732664121555/Put/seqid=0 2024-11-26T23:35:21,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742288_1464 (size=12151) 2024-11-26T23:35:22,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T23:35:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T23:35:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664182252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664182252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664182252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664182261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T23:35:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664182271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/8b92a36e5a8a49b1afea4c98fa2d4286 2024-11-26T23:35:22,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/14c9ee6387714cb1a70f7b19dd389c54 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/14c9ee6387714cb1a70f7b19dd389c54 2024-11-26T23:35:22,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/14c9ee6387714cb1a70f7b19dd389c54, entries=250, sequenceid=253, filesize=16.5 K 2024-11-26T23:35:22,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/5fcf46b740544b97aada09dc60abc28a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5fcf46b740544b97aada09dc60abc28a 2024-11-26T23:35:22,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5fcf46b740544b97aada09dc60abc28a, entries=150, sequenceid=253, filesize=11.9 K 2024-11-26T23:35:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/8b92a36e5a8a49b1afea4c98fa2d4286 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/8b92a36e5a8a49b1afea4c98fa2d4286 2024-11-26T23:35:22,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/8b92a36e5a8a49b1afea4c98fa2d4286, entries=150, sequenceid=253, filesize=11.9 K 2024-11-26T23:35:22,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 88d8390f4cce6ec80971092bc0ed5e72 in 850ms, sequenceid=253, compaction requested=true 2024-11-26T23:35:22,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:22,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:22,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:22,405 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:22,405 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:22,407 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:22,407 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:22,407 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:22,407 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:22,407 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,407 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,407 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/67798ab9617e45479627901a0f70d0d2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b4a5bf45068b4b619a7dc59e12abda82, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e2744457d1a84c22bbc17551ddaa8709, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5fcf46b740544b97aada09dc60abc28a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=47.9 K 2024-11-26T23:35:22,407 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/743d1d6a7a5e4da7a908a6fa73831e05, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/043b67c9e0e848c2914c0a3f10101de7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/157bcc315e8e469cbea6aa7e24591d5d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/14c9ee6387714cb1a70f7b19dd389c54] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=54.9 K 2024-11-26T23:35:22,407 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 67798ab9617e45479627901a0f70d0d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664119223 2024-11-26T23:35:22,407 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 743d1d6a7a5e4da7a908a6fa73831e05, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664119223 2024-11-26T23:35:22,407 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b4a5bf45068b4b619a7dc59e12abda82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732664119344 2024-11-26T23:35:22,408 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 043b67c9e0e848c2914c0a3f10101de7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732664119344 2024-11-26T23:35:22,408 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e2744457d1a84c22bbc17551ddaa8709, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732664120488 2024-11-26T23:35:22,408 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 157bcc315e8e469cbea6aa7e24591d5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732664120488 2024-11-26T23:35:22,408 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fcf46b740544b97aada09dc60abc28a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664121219 2024-11-26T23:35:22,408 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c9ee6387714cb1a70f7b19dd389c54, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664121188 2024-11-26T23:35:22,415 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:22,415 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:22,416 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/2241dad4c703496a842cedca55394885 is 50, key is test_row_0/B:col10/1732664121555/Put/seqid=0 2024-11-26T23:35:22,416 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/98cec3537461456a97bcb37e78018d5c is 50, key is test_row_0/A:col10/1732664121555/Put/seqid=0 2024-11-26T23:35:22,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742290_1466 (size=12731) 2024-11-26T23:35:22,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742289_1465 (size=12731) 2024-11-26T23:35:22,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T23:35:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-26T23:35:22,429 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/2241dad4c703496a842cedca55394885 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/2241dad4c703496a842cedca55394885 2024-11-26T23:35:22,429 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/98cec3537461456a97bcb37e78018d5c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/98cec3537461456a97bcb37e78018d5c 2024-11-26T23:35:22,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/2108cc83bbe349c3b0ca580305ff210f is 50, key is test_row_0/A:col10/1732664121588/Put/seqid=0 2024-11-26T23:35:22,432 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 2241dad4c703496a842cedca55394885(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:22,432 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:22,432 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=12, startTime=1732664122405; duration=0sec 2024-11-26T23:35:22,432 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:22,432 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:22,432 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:22,433 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into 98cec3537461456a97bcb37e78018d5c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:22,433 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:22,433 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=12, startTime=1732664122405; duration=0sec 2024-11-26T23:35:22,433 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:22,433 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:22,434 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:22,434 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:22,434 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:22,434 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/4aced4302a1e43d2bfbf16585bb989ef, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/d0e3c0b21eb84a308cd25e6fe437561a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/98e8037f92514a42a08db9f06978177e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/8b92a36e5a8a49b1afea4c98fa2d4286] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=47.9 K 2024-11-26T23:35:22,434 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aced4302a1e43d2bfbf16585bb989ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732664119223 2024-11-26T23:35:22,434 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e3c0b21eb84a308cd25e6fe437561a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732664119344 2024-11-26T23:35:22,435 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e8037f92514a42a08db9f06978177e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732664120488 2024-11-26T23:35:22,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742291_1467 (size=12301) 2024-11-26T23:35:22,436 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b92a36e5a8a49b1afea4c98fa2d4286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664121219 2024-11-26T23:35:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T23:35:22,443 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:22,444 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/450305a54508484691f57bdd4f06da58 is 50, key is test_row_0/C:col10/1732664121555/Put/seqid=0 2024-11-26T23:35:22,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742292_1468 (size=12731) 2024-11-26T23:35:22,456 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/450305a54508484691f57bdd4f06da58 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/450305a54508484691f57bdd4f06da58 2024-11-26T23:35:22,459 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into 450305a54508484691f57bdd4f06da58(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:22,459 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:22,459 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=12, startTime=1732664122406; duration=0sec 2024-11-26T23:35:22,459 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:22,459 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:22,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664182790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664182791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664182792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664182802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664182802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/2108cc83bbe349c3b0ca580305ff210f 2024-11-26T23:35:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/91cef52f710840369b7817e3096cbfb2 is 50, key is test_row_0/B:col10/1732664121588/Put/seqid=0 2024-11-26T23:35:22,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742293_1469 (size=12301) 2024-11-26T23:35:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664182911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664182912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664182912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664182912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664182912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T23:35:23,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664183122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664183131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664183131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664183131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664183132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/91cef52f710840369b7817e3096cbfb2 2024-11-26T23:35:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/80bc7b8a23734f78b4d4395ae940db93 is 50, key is test_row_0/C:col10/1732664121588/Put/seqid=0 2024-11-26T23:35:23,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742294_1470 (size=12301) 2024-11-26T23:35:23,278 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/80bc7b8a23734f78b4d4395ae940db93 2024-11-26T23:35:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/2108cc83bbe349c3b0ca580305ff210f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/2108cc83bbe349c3b0ca580305ff210f 2024-11-26T23:35:23,293 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/2108cc83bbe349c3b0ca580305ff210f, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T23:35:23,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/91cef52f710840369b7817e3096cbfb2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/91cef52f710840369b7817e3096cbfb2 2024-11-26T23:35:23,297 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/91cef52f710840369b7817e3096cbfb2, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T23:35:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/80bc7b8a23734f78b4d4395ae940db93 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/80bc7b8a23734f78b4d4395ae940db93 2024-11-26T23:35:23,308 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/80bc7b8a23734f78b4d4395ae940db93, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T23:35:23,309 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 88d8390f4cce6ec80971092bc0ed5e72 in 883ms, sequenceid=275, compaction requested=false 2024-11-26T23:35:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-26T23:35:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-26T23:35:23,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-26T23:35:23,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4960 sec 2024-11-26T23:35:23,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5030 sec 2024-11-26T23:35:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:23,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-26T23:35:23,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/794b1a0103d747e0a58d9e9969831223 is 50, key is test_row_0/A:col10/1732664123443/Put/seqid=0 2024-11-26T23:35:23,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742295_1471 (size=14741) 2024-11-26T23:35:23,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664183482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664183492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664183492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664183494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664183496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664183601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664183611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664183612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664183612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664183612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664183821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664183832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664183832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664183832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664183832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:23,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/794b1a0103d747e0a58d9e9969831223 2024-11-26T23:35:23,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0d7ba81520c84b4c8a0ca74514902df9 is 50, key is test_row_0/B:col10/1732664123443/Put/seqid=0 2024-11-26T23:35:23,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742296_1472 (size=12301) 2024-11-26T23:35:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T23:35:23,961 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:23,962 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-26T23:35:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T23:35:23,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:23,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:23,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T23:35:24,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T23:35:24,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:24,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664184131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664184152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664184152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664184152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664184152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T23:35:24,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:24,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T23:35:24,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0d7ba81520c84b4c8a0ca74514902df9 2024-11-26T23:35:24,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/833fd88e610f4fc6b789293e3ca4904a is 50, key is test_row_0/C:col10/1732664123443/Put/seqid=0 2024-11-26T23:35:24,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742297_1473 (size=12301) 2024-11-26T23:35:24,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T23:35:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T23:35:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T23:35:24,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664184652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664184662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664184662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664184671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664184671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:24,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/833fd88e610f4fc6b789293e3ca4904a 2024-11-26T23:35:24,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/794b1a0103d747e0a58d9e9969831223 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/794b1a0103d747e0a58d9e9969831223 2024-11-26T23:35:24,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/794b1a0103d747e0a58d9e9969831223, entries=200, sequenceid=294, filesize=14.4 K 2024-11-26T23:35:24,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0d7ba81520c84b4c8a0ca74514902df9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0d7ba81520c84b4c8a0ca74514902df9 2024-11-26T23:35:24,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0d7ba81520c84b4c8a0ca74514902df9, entries=150, sequenceid=294, filesize=12.0 K 2024-11-26T23:35:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/833fd88e610f4fc6b789293e3ca4904a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/833fd88e610f4fc6b789293e3ca4904a 2024-11-26T23:35:24,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/833fd88e610f4fc6b789293e3ca4904a, entries=150, sequenceid=294, filesize=12.0 K 2024-11-26T23:35:24,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 88d8390f4cce6ec80971092bc0ed5e72 in 1277ms, sequenceid=294, compaction requested=true 2024-11-26T23:35:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:24,723 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:24,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:24,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T23:35:24,724 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:24,724 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:24,724 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:24,724 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,725 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:24,725 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/98cec3537461456a97bcb37e78018d5c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/2108cc83bbe349c3b0ca580305ff210f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/794b1a0103d747e0a58d9e9969831223] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=38.8 K 2024-11-26T23:35:24,725 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:24,725 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,725 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/2241dad4c703496a842cedca55394885, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/91cef52f710840369b7817e3096cbfb2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0d7ba81520c84b4c8a0ca74514902df9] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.5 K 2024-11-26T23:35:24,725 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98cec3537461456a97bcb37e78018d5c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664121219 2024-11-26T23:35:24,725 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 2241dad4c703496a842cedca55394885, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664121219 2024-11-26T23:35:24,725 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 91cef52f710840369b7817e3096cbfb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732664121588 2024-11-26T23:35:24,726 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2108cc83bbe349c3b0ca580305ff210f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732664121588 2024-11-26T23:35:24,726 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 794b1a0103d747e0a58d9e9969831223, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732664122780 2024-11-26T23:35:24,726 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d7ba81520c84b4c8a0ca74514902df9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732664122780 2024-11-26T23:35:24,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T23:35:24,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:24,729 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-26T23:35:24,740 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:24,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/82196dfd53a14790bfcd4c15ad431a1f is 50, key is test_row_0/A:col10/1732664123478/Put/seqid=0 2024-11-26T23:35:24,741 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/82083e30f26f4a48b92ad69ebdfd4b68 is 50, key is test_row_0/B:col10/1732664123443/Put/seqid=0 2024-11-26T23:35:24,742 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:24,743 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/b0fe8e5ca6bd4327b6c6db4956a75743 is 50, key is test_row_0/A:col10/1732664123443/Put/seqid=0 2024-11-26T23:35:24,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742298_1474 (size=12301) 2024-11-26T23:35:24,760 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/82196dfd53a14790bfcd4c15ad431a1f 2024-11-26T23:35:24,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742299_1475 (size=12983) 2024-11-26T23:35:24,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742300_1476 (size=12983) 2024-11-26T23:35:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/a33ae26efd3d4f90931305b1dd04436d is 50, key is test_row_0/B:col10/1732664123478/Put/seqid=0 2024-11-26T23:35:24,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742301_1477 (size=12301) 2024-11-26T23:35:24,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/a33ae26efd3d4f90931305b1dd04436d 2024-11-26T23:35:24,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b774bb8d8c0b44fd9062bd248895ed0c is 50, key is test_row_0/C:col10/1732664123478/Put/seqid=0 2024-11-26T23:35:24,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742302_1478 (size=12301) 2024-11-26T23:35:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T23:35:25,174 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/82083e30f26f4a48b92ad69ebdfd4b68 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/82083e30f26f4a48b92ad69ebdfd4b68 2024-11-26T23:35:25,179 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 82083e30f26f4a48b92ad69ebdfd4b68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:25,179 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:25,179 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=13, startTime=1732664124723; duration=0sec 2024-11-26T23:35:25,179 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:25,179 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:25,179 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:25,181 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:25,181 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:25,181 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:25,181 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/450305a54508484691f57bdd4f06da58, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/80bc7b8a23734f78b4d4395ae940db93, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/833fd88e610f4fc6b789293e3ca4904a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.5 K 2024-11-26T23:35:25,182 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 450305a54508484691f57bdd4f06da58, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732664121219 2024-11-26T23:35:25,182 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 80bc7b8a23734f78b4d4395ae940db93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732664121588 2024-11-26T23:35:25,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 833fd88e610f4fc6b789293e3ca4904a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732664122780 2024-11-26T23:35:25,183 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/b0fe8e5ca6bd4327b6c6db4956a75743 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b0fe8e5ca6bd4327b6c6db4956a75743 2024-11-26T23:35:25,187 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into b0fe8e5ca6bd4327b6c6db4956a75743(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:25,187 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:25,187 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=13, startTime=1732664124723; duration=0sec 2024-11-26T23:35:25,187 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:25,187 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:25,188 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:25,188 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/2e3c332666a24b1ab3f26ab5e920df8f is 50, key is test_row_0/C:col10/1732664123443/Put/seqid=0 2024-11-26T23:35:25,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742303_1479 (size=12983) 2024-11-26T23:35:25,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b774bb8d8c0b44fd9062bd248895ed0c 2024-11-26T23:35:25,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/82196dfd53a14790bfcd4c15ad431a1f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/82196dfd53a14790bfcd4c15ad431a1f 2024-11-26T23:35:25,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/82196dfd53a14790bfcd4c15ad431a1f, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T23:35:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/a33ae26efd3d4f90931305b1dd04436d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/a33ae26efd3d4f90931305b1dd04436d 2024-11-26T23:35:25,247 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/a33ae26efd3d4f90931305b1dd04436d, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T23:35:25,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b774bb8d8c0b44fd9062bd248895ed0c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b774bb8d8c0b44fd9062bd248895ed0c 2024-11-26T23:35:25,251 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b774bb8d8c0b44fd9062bd248895ed0c, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T23:35:25,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 88d8390f4cce6ec80971092bc0ed5e72 in 522ms, sequenceid=314, compaction requested=false 2024-11-26T23:35:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-26T23:35:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-26T23:35:25,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-26T23:35:25,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2890 sec 2024-11-26T23:35:25,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2940 sec 2024-11-26T23:35:25,600 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/2e3c332666a24b1ab3f26ab5e920df8f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2e3c332666a24b1ab3f26ab5e920df8f 2024-11-26T23:35:25,604 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into 2e3c332666a24b1ab3f26ab5e920df8f(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:25,604 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:25,604 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=13, startTime=1732664124723; duration=0sec 2024-11-26T23:35:25,604 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:25,604 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:25,689 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:25,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:35:25,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7fdbb076e8064022acff91c6add487ac is 50, key is test_row_0/A:col10/1732664125687/Put/seqid=0 2024-11-26T23:35:25,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742304_1480 (size=14741) 2024-11-26T23:35:25,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664185783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664185785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664185785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664185794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664185801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664185901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664185911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664185911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664185911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:25,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664185922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7fdbb076e8064022acff91c6add487ac 2024-11-26T23:35:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T23:35:26,112 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:26,113 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-26T23:35:26,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664186111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,117 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T23:35:26,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:26,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:26,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/5082927db7394c57bb1f937252748ac5 is 50, key is test_row_0/B:col10/1732664125687/Put/seqid=0 2024-11-26T23:35:26,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664186132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664186132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664186132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664186141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742305_1481 (size=12301) 2024-11-26T23:35:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T23:35:26,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T23:35:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T23:35:26,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:26,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T23:35:26,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664186422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664186451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664186451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664186451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664186452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/5082927db7394c57bb1f937252748ac5 2024-11-26T23:35:26,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/29d5544000924b4e81db99f9e6e51f42 is 50, key is test_row_0/C:col10/1732664125687/Put/seqid=0 2024-11-26T23:35:26,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742306_1482 (size=12301) 2024-11-26T23:35:26,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T23:35:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T23:35:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T23:35:26,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T23:35:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:26,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664186941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/29d5544000924b4e81db99f9e6e51f42 2024-11-26T23:35:26,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/7fdbb076e8064022acff91c6add487ac as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7fdbb076e8064022acff91c6add487ac 2024-11-26T23:35:26,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7fdbb076e8064022acff91c6add487ac, entries=200, sequenceid=328, filesize=14.4 K 2024-11-26T23:35:26,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/5082927db7394c57bb1f937252748ac5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5082927db7394c57bb1f937252748ac5 2024-11-26T23:35:26,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5082927db7394c57bb1f937252748ac5, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T23:35:26,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/29d5544000924b4e81db99f9e6e51f42 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/29d5544000924b4e81db99f9e6e51f42 2024-11-26T23:35:26,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664186972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664186972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664186972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664186972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:26,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/29d5544000924b4e81db99f9e6e51f42, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T23:35:26,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 88d8390f4cce6ec80971092bc0ed5e72 in 1295ms, sequenceid=328, compaction requested=true 2024-11-26T23:35:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:26,984 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:26,984 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:26,985 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,985 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:26,985 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/82083e30f26f4a48b92ad69ebdfd4b68, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/a33ae26efd3d4f90931305b1dd04436d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5082927db7394c57bb1f937252748ac5] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.7 K 2024-11-26T23:35:26,985 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b0fe8e5ca6bd4327b6c6db4956a75743, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/82196dfd53a14790bfcd4c15ad431a1f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7fdbb076e8064022acff91c6add487ac] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=39.1 K 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0fe8e5ca6bd4327b6c6db4956a75743, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732664122780 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 82083e30f26f4a48b92ad69ebdfd4b68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732664122780 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82196dfd53a14790bfcd4c15ad431a1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732664123478 2024-11-26T23:35:26,985 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting a33ae26efd3d4f90931305b1dd04436d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732664123478 2024-11-26T23:35:26,986 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fdbb076e8064022acff91c6add487ac, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664125673 2024-11-26T23:35:26,986 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5082927db7394c57bb1f937252748ac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664125673 2024-11-26T23:35:26,992 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:26,992 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/90f20afe05f34fa8a6ce9644bb0e14fe is 50, key is test_row_0/B:col10/1732664125687/Put/seqid=0 2024-11-26T23:35:26,992 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:26,993 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/dd4b8aeb37b2451a8aa00f9b4b93c7b1 is 50, key is test_row_0/A:col10/1732664125687/Put/seqid=0 2024-11-26T23:35:26,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742307_1483 (size=13085) 2024-11-26T23:35:26,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742308_1484 (size=13085) 2024-11-26T23:35:27,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T23:35:27,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:27,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T23:35:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/58685bffa86e458bb7839788c4235b06 is 50, key is test_row_0/A:col10/1732664125782/Put/seqid=0 2024-11-26T23:35:27,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742309_1485 (size=12301) 2024-11-26T23:35:27,047 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/58685bffa86e458bb7839788c4235b06 2024-11-26T23:35:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6fb4d266fa3b45e5b83a5086439c15ee is 50, key is test_row_0/B:col10/1732664125782/Put/seqid=0 2024-11-26T23:35:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742310_1486 (size=12301) 2024-11-26T23:35:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T23:35:27,401 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/dd4b8aeb37b2451a8aa00f9b4b93c7b1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/dd4b8aeb37b2451a8aa00f9b4b93c7b1 2024-11-26T23:35:27,402 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/90f20afe05f34fa8a6ce9644bb0e14fe as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/90f20afe05f34fa8a6ce9644bb0e14fe 2024-11-26T23:35:27,407 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 90f20afe05f34fa8a6ce9644bb0e14fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:27,407 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into dd4b8aeb37b2451a8aa00f9b4b93c7b1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:27,407 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:27,407 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:27,407 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=13, startTime=1732664126984; duration=0sec 2024-11-26T23:35:27,407 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=13, startTime=1732664126984; duration=0sec 2024-11-26T23:35:27,408 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:27,408 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:27,408 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:27,408 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:27,408 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:27,408 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:27,409 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:27,409 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:27,409 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2e3c332666a24b1ab3f26ab5e920df8f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b774bb8d8c0b44fd9062bd248895ed0c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/29d5544000924b4e81db99f9e6e51f42] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.7 K 2024-11-26T23:35:27,409 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e3c332666a24b1ab3f26ab5e920df8f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732664122780 2024-11-26T23:35:27,409 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b774bb8d8c0b44fd9062bd248895ed0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732664123478 2024-11-26T23:35:27,410 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d5544000924b4e81db99f9e6e51f42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664125673 2024-11-26T23:35:27,415 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:27,416 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/38d5a308585b421e8b6937456e0d2022 is 50, key is test_row_0/C:col10/1732664125687/Put/seqid=0 2024-11-26T23:35:27,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742311_1487 (size=13085) 2024-11-26T23:35:27,458 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6fb4d266fa3b45e5b83a5086439c15ee 2024-11-26T23:35:27,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b3e49f19897741f2a63058bdd5e81faf is 50, key is test_row_0/C:col10/1732664125782/Put/seqid=0 2024-11-26T23:35:27,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742312_1488 (size=12301) 2024-11-26T23:35:27,825 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/38d5a308585b421e8b6937456e0d2022 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/38d5a308585b421e8b6937456e0d2022 2024-11-26T23:35:27,829 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into 38d5a308585b421e8b6937456e0d2022(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:27,829 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:27,829 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=13, startTime=1732664126984; duration=0sec 2024-11-26T23:35:27,830 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:27,830 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:27,877 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b3e49f19897741f2a63058bdd5e81faf 2024-11-26T23:35:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/58685bffa86e458bb7839788c4235b06 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58685bffa86e458bb7839788c4235b06 2024-11-26T23:35:27,894 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58685bffa86e458bb7839788c4235b06, entries=150, sequenceid=353, filesize=12.0 K 2024-11-26T23:35:27,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6fb4d266fa3b45e5b83a5086439c15ee as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6fb4d266fa3b45e5b83a5086439c15ee 2024-11-26T23:35:27,898 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6fb4d266fa3b45e5b83a5086439c15ee, entries=150, sequenceid=353, filesize=12.0 K 2024-11-26T23:35:27,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b3e49f19897741f2a63058bdd5e81faf as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b3e49f19897741f2a63058bdd5e81faf 2024-11-26T23:35:27,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b3e49f19897741f2a63058bdd5e81faf, entries=150, sequenceid=353, filesize=12.0 K 2024-11-26T23:35:27,904 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 88d8390f4cce6ec80971092bc0ed5e72 in 866ms, sequenceid=353, compaction requested=false 2024-11-26T23:35:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-26T23:35:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-26T23:35:27,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-26T23:35:27,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7900 sec 2024-11-26T23:35:27,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7990 sec 2024-11-26T23:35:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:27,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:27,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:35:27,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/58115b9822df4843a1ae6e5f9c9432fc is 50, key is test_row_0/A:col10/1732664127980/Put/seqid=0 2024-11-26T23:35:27,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742313_1489 (size=17181) 2024-11-26T23:35:28,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664188045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664188046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664188077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664188077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664188078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664188182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664188182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664188201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664188201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664188201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T23:35:28,261 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:28,262 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-26T23:35:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T23:35:28,264 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:28,265 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:28,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T23:35:28,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/58115b9822df4843a1ae6e5f9c9432fc 2024-11-26T23:35:28,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/ea26fca4fcfd4fae90aad89418ac1e05 is 50, key is test_row_0/B:col10/1732664127980/Put/seqid=0 2024-11-26T23:35:28,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742314_1490 (size=12301) 2024-11-26T23:35:28,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/ea26fca4fcfd4fae90aad89418ac1e05 2024-11-26T23:35:28,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b20e892a758149e1ad84729406d46e81 is 50, key is test_row_0/C:col10/1732664127980/Put/seqid=0 2024-11-26T23:35:28,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742315_1491 (size=12301) 2024-11-26T23:35:28,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664188402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664188402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T23:35:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664188421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664188421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664188421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T23:35:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T23:35:28,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T23:35:28,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:28,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:28,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664188721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664188721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664188731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664188732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664188732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:28,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b20e892a758149e1ad84729406d46e81 2024-11-26T23:35:28,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/58115b9822df4843a1ae6e5f9c9432fc as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58115b9822df4843a1ae6e5f9c9432fc 2024-11-26T23:35:28,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58115b9822df4843a1ae6e5f9c9432fc, entries=250, sequenceid=367, filesize=16.8 K 2024-11-26T23:35:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/ea26fca4fcfd4fae90aad89418ac1e05 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/ea26fca4fcfd4fae90aad89418ac1e05 2024-11-26T23:35:28,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/ea26fca4fcfd4fae90aad89418ac1e05, entries=150, sequenceid=367, filesize=12.0 K 2024-11-26T23:35:28,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b20e892a758149e1ad84729406d46e81 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b20e892a758149e1ad84729406d46e81 2024-11-26T23:35:28,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b20e892a758149e1ad84729406d46e81, entries=150, sequenceid=367, filesize=12.0 K 2024-11-26T23:35:28,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 88d8390f4cce6ec80971092bc0ed5e72 in 845ms, sequenceid=367, compaction requested=true 2024-11-26T23:35:28,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:28,826 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:28,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:28,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:28,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:28,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:28,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:28,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:28,827 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:28,828 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:28,828 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:28,828 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,828 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/90f20afe05f34fa8a6ce9644bb0e14fe, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6fb4d266fa3b45e5b83a5086439c15ee, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/ea26fca4fcfd4fae90aad89418ac1e05] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.8 K 2024-11-26T23:35:28,828 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:28,828 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:28,828 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,828 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/dd4b8aeb37b2451a8aa00f9b4b93c7b1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58685bffa86e458bb7839788c4235b06, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58115b9822df4843a1ae6e5f9c9432fc] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=41.6 K 2024-11-26T23:35:28,829 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd4b8aeb37b2451a8aa00f9b4b93c7b1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664125673 2024-11-26T23:35:28,829 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f20afe05f34fa8a6ce9644bb0e14fe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664125673 2024-11-26T23:35:28,829 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58685bffa86e458bb7839788c4235b06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732664125779 2024-11-26T23:35:28,829 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fb4d266fa3b45e5b83a5086439c15ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732664125779 2024-11-26T23:35:28,829 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58115b9822df4843a1ae6e5f9c9432fc, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664127962 2024-11-26T23:35:28,830 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting ea26fca4fcfd4fae90aad89418ac1e05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664127966 2024-11-26T23:35:28,846 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:28,847 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/97cba938af284a388f8b6b950da206b3 is 50, key is test_row_0/B:col10/1732664127980/Put/seqid=0 2024-11-26T23:35:28,854 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:28,855 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/80303ebb26ef4bd6b6fb72f337b1a301 is 50, key is test_row_0/A:col10/1732664127980/Put/seqid=0 2024-11-26T23:35:28,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742316_1492 (size=13187) 2024-11-26T23:35:28,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742317_1493 (size=13187) 2024-11-26T23:35:28,872 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/97cba938af284a388f8b6b950da206b3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/97cba938af284a388f8b6b950da206b3 2024-11-26T23:35:28,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T23:35:28,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,875 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:35:28,877 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 97cba938af284a388f8b6b950da206b3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:28,877 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:28,877 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=13, startTime=1732664128826; duration=0sec 2024-11-26T23:35:28,877 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:28,877 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:28,877 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:28,879 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:28,879 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:28,879 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:28,879 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/38d5a308585b421e8b6937456e0d2022, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b3e49f19897741f2a63058bdd5e81faf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b20e892a758149e1ad84729406d46e81] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.8 K 2024-11-26T23:35:28,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/8e293232a3404ea3958847306b1a10f8 is 50, key is test_row_0/A:col10/1732664128054/Put/seqid=0 2024-11-26T23:35:28,882 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d5a308585b421e8b6937456e0d2022, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732664125673 2024-11-26T23:35:28,882 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e49f19897741f2a63058bdd5e81faf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732664125779 2024-11-26T23:35:28,883 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b20e892a758149e1ad84729406d46e81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664127966 2024-11-26T23:35:28,891 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:28,891 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b1b4c51e8fc64fe99af4810427564d86 is 50, key is test_row_0/C:col10/1732664127980/Put/seqid=0 2024-11-26T23:35:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T23:35:28,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742318_1494 (size=12301) 2024-11-26T23:35:28,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742319_1495 (size=13187) 2024-11-26T23:35:28,908 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/b1b4c51e8fc64fe99af4810427564d86 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b1b4c51e8fc64fe99af4810427564d86 2024-11-26T23:35:28,912 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into b1b4c51e8fc64fe99af4810427564d86(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:28,912 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:28,912 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=13, startTime=1732664128826; duration=0sec 2024-11-26T23:35:28,912 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:28,912 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664189253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664189262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664189262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664189263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664189263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,278 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/80303ebb26ef4bd6b6fb72f337b1a301 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/80303ebb26ef4bd6b6fb72f337b1a301 2024-11-26T23:35:29,282 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into 80303ebb26ef4bd6b6fb72f337b1a301(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:29,282 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:29,282 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=13, startTime=1732664128826; duration=0sec 2024-11-26T23:35:29,282 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:29,282 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:29,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/8e293232a3404ea3958847306b1a10f8 2024-11-26T23:35:29,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e5fff50715964531bc94dec3b5cea704 is 50, key is test_row_0/B:col10/1732664128054/Put/seqid=0 2024-11-26T23:35:29,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742320_1496 (size=12301) 2024-11-26T23:35:29,315 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e5fff50715964531bc94dec3b5cea704 2024-11-26T23:35:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/ba220090f572493b94abc5423b055d3a is 50, key is test_row_0/C:col10/1732664128054/Put/seqid=0 2024-11-26T23:35:29,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742321_1497 (size=12301) 2024-11-26T23:35:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664189371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664189371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664189381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664189382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664189382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T23:35:29,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664189581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664189581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664189591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664189602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664189602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,734 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/ba220090f572493b94abc5423b055d3a 2024-11-26T23:35:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/8e293232a3404ea3958847306b1a10f8 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/8e293232a3404ea3958847306b1a10f8 2024-11-26T23:35:29,741 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/8e293232a3404ea3958847306b1a10f8, entries=150, sequenceid=394, filesize=12.0 K 2024-11-26T23:35:29,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/e5fff50715964531bc94dec3b5cea704 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e5fff50715964531bc94dec3b5cea704 2024-11-26T23:35:29,744 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e5fff50715964531bc94dec3b5cea704, entries=150, sequenceid=394, filesize=12.0 K 2024-11-26T23:35:29,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/ba220090f572493b94abc5423b055d3a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/ba220090f572493b94abc5423b055d3a 2024-11-26T23:35:29,748 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/ba220090f572493b94abc5423b055d3a, entries=150, sequenceid=394, filesize=12.0 K 2024-11-26T23:35:29,748 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 88d8390f4cce6ec80971092bc0ed5e72 in 873ms, sequenceid=394, compaction requested=false 2024-11-26T23:35:29,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:29,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:29,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-26T23:35:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-26T23:35:29,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-26T23:35:29,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-11-26T23:35:29,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4890 sec 2024-11-26T23:35:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:29,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:29,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:35:29,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/d5546e8327ca478eb7ba6f756b8c434a is 50, key is test_row_0/A:col10/1732664129261/Put/seqid=0 2024-11-26T23:35:29,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742322_1498 (size=14741) 2024-11-26T23:35:29,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664189954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664189954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664189955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664189967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:29,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664189967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664190071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664190071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664190071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664190082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664190092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664190281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664190281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664190281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664190302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/d5546e8327ca478eb7ba6f756b8c434a 2024-11-26T23:35:30,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664190312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/211c13602ef949dd8fb9549d9ae31415 is 50, key is test_row_0/B:col10/1732664129261/Put/seqid=0 2024-11-26T23:35:30,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742323_1499 (size=12301) 2024-11-26T23:35:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T23:35:30,412 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:30,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-26T23:35:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T23:35:30,414 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:30,414 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:30,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T23:35:30,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:30,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664190601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664190601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664190601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664190622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664190632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:30,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:30,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:30,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:30,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:30,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/211c13602ef949dd8fb9549d9ae31415 2024-11-26T23:35:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T23:35:30,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/698a7b7b99a344ef8e984216112af812 is 50, key is test_row_0/C:col10/1732664129261/Put/seqid=0 2024-11-26T23:35:30,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742324_1500 (size=12301) 2024-11-26T23:35:30,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:30,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:30,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:30,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:30,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T23:35:31,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664191121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664191122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:31,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664191122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:31,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/698a7b7b99a344ef8e984216112af812 2024-11-26T23:35:31,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/d5546e8327ca478eb7ba6f756b8c434a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d5546e8327ca478eb7ba6f756b8c434a 2024-11-26T23:35:31,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d5546e8327ca478eb7ba6f756b8c434a, entries=200, sequenceid=408, filesize=14.4 K 2024-11-26T23:35:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/211c13602ef949dd8fb9549d9ae31415 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/211c13602ef949dd8fb9549d9ae31415 2024-11-26T23:35:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664191131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:31,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/211c13602ef949dd8fb9549d9ae31415, entries=150, sequenceid=408, filesize=12.0 K 2024-11-26T23:35:31,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/698a7b7b99a344ef8e984216112af812 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/698a7b7b99a344ef8e984216112af812 2024-11-26T23:35:31,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/698a7b7b99a344ef8e984216112af812, entries=150, sequenceid=408, filesize=12.0 K 2024-11-26T23:35:31,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 88d8390f4cce6ec80971092bc0ed5e72 in 1256ms, sequenceid=408, compaction requested=true 2024-11-26T23:35:31,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:31,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:31,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:31,151 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:31,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:31,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:31,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:31,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:31,151 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:31,152 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:31,152 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:31,152 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,152 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:31,152 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/80303ebb26ef4bd6b6fb72f337b1a301, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/8e293232a3404ea3958847306b1a10f8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d5546e8327ca478eb7ba6f756b8c434a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=39.3 K 2024-11-26T23:35:31,152 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:31,152 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,152 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/97cba938af284a388f8b6b950da206b3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e5fff50715964531bc94dec3b5cea704, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/211c13602ef949dd8fb9549d9ae31415] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.9 K 2024-11-26T23:35:31,152 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80303ebb26ef4bd6b6fb72f337b1a301, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664127966 2024-11-26T23:35:31,152 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 97cba938af284a388f8b6b950da206b3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664127966 2024-11-26T23:35:31,152 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e293232a3404ea3958847306b1a10f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732664128034 2024-11-26T23:35:31,153 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e5fff50715964531bc94dec3b5cea704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732664128034 2024-11-26T23:35:31,153 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5546e8327ca478eb7ba6f756b8c434a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732664129249 2024-11-26T23:35:31,153 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 211c13602ef949dd8fb9549d9ae31415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732664129249 2024-11-26T23:35:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:31,159 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:31,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T23:35:31,161 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:31,161 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6d0444b4fec54009a9fae3485c0d517b is 50, key is test_row_0/B:col10/1732664129261/Put/seqid=0 2024-11-26T23:35:31,163 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:31,163 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/f169d461f3c142278567367ba1efc149 is 50, key is test_row_0/A:col10/1732664129261/Put/seqid=0 2024-11-26T23:35:31,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/73f86b5282db4f70882b45b2e95edd15 is 50, key is test_row_0/A:col10/1732664131158/Put/seqid=0 2024-11-26T23:35:31,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742325_1501 (size=13289) 2024-11-26T23:35:31,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742326_1502 (size=14741) 2024-11-26T23:35:31,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,183 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/6d0444b4fec54009a9fae3485c0d517b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6d0444b4fec54009a9fae3485c0d517b 2024-11-26T23:35:31,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/73f86b5282db4f70882b45b2e95edd15 2024-11-26T23:35:31,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742327_1503 (size=13289) 2024-11-26T23:35:31,189 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 6d0444b4fec54009a9fae3485c0d517b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:31,189 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:31,189 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=13, startTime=1732664131151; duration=0sec 2024-11-26T23:35:31,189 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:31,189 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:31,189 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:31,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/b3a0338654d643e58f50cf3884540e7e is 50, key is test_row_0/B:col10/1732664131158/Put/seqid=0 2024-11-26T23:35:31,193 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:31,193 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/C is initiating minor compaction (all files) 2024-11-26T23:35:31,193 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/C in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,193 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b1b4c51e8fc64fe99af4810427564d86, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/ba220090f572493b94abc5423b055d3a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/698a7b7b99a344ef8e984216112af812] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=36.9 K 2024-11-26T23:35:31,193 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b4c51e8fc64fe99af4810427564d86, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732664127966 2024-11-26T23:35:31,193 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting ba220090f572493b94abc5423b055d3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732664128034 2024-11-26T23:35:31,194 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 698a7b7b99a344ef8e984216112af812, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732664129249 2024-11-26T23:35:31,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742328_1504 (size=12301) 2024-11-26T23:35:31,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/b3a0338654d643e58f50cf3884540e7e 2024-11-26T23:35:31,201 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:31,201 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/7ebedfe18ba247cb90dc846ae11f089c is 50, key is test_row_0/C:col10/1732664129261/Put/seqid=0 2024-11-26T23:35:31,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/f5d0ca4c302d40219700748f87bf4166 is 50, key is test_row_0/C:col10/1732664131158/Put/seqid=0 2024-11-26T23:35:31,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742329_1505 (size=13289) 2024-11-26T23:35:31,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742330_1506 (size=12301) 2024-11-26T23:35:31,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664191212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664191321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:31,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:31,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:31,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:31,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:31,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664191532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T23:35:31,584 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/f169d461f3c142278567367ba1efc149 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f169d461f3c142278567367ba1efc149 2024-11-26T23:35:31,588 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into f169d461f3c142278567367ba1efc149(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:31,588 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:31,588 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=13, startTime=1732664131151; duration=0sec 2024-11-26T23:35:31,588 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:31,588 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:31,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/f5d0ca4c302d40219700748f87bf4166 2024-11-26T23:35:31,609 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/7ebedfe18ba247cb90dc846ae11f089c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/7ebedfe18ba247cb90dc846ae11f089c 2024-11-26T23:35:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/73f86b5282db4f70882b45b2e95edd15 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/73f86b5282db4f70882b45b2e95edd15 2024-11-26T23:35:31,613 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/C of 88d8390f4cce6ec80971092bc0ed5e72 into 7ebedfe18ba247cb90dc846ae11f089c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:31,613 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:31,613 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/C, priority=13, startTime=1732664131151; duration=0sec 2024-11-26T23:35:31,613 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:31,613 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:31,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/73f86b5282db4f70882b45b2e95edd15, entries=200, sequenceid=435, filesize=14.4 K 2024-11-26T23:35:31,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/b3a0338654d643e58f50cf3884540e7e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b3a0338654d643e58f50cf3884540e7e 2024-11-26T23:35:31,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b3a0338654d643e58f50cf3884540e7e, entries=150, sequenceid=435, filesize=12.0 K 2024-11-26T23:35:31,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/f5d0ca4c302d40219700748f87bf4166 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f5d0ca4c302d40219700748f87bf4166 2024-11-26T23:35:31,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f5d0ca4c302d40219700748f87bf4166, entries=150, sequenceid=435, filesize=12.0 K 2024-11-26T23:35:31,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 88d8390f4cce6ec80971092bc0ed5e72 in 463ms, sequenceid=435, compaction requested=false 2024-11-26T23:35:31,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:31,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T23:35:31,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:31,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-26T23:35:31,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/60a7328ebb4942139a65f698c2be6304 is 50, key is test_row_0/A:col10/1732664131199/Put/seqid=0 2024-11-26T23:35:31,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742331_1507 (size=12301) 2024-11-26T23:35:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:31,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. as already flushing 2024-11-26T23:35:32,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/60a7328ebb4942139a65f698c2be6304 2024-11-26T23:35:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664192052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/24a26b4f022e4d5ea10ee1659431ca16 is 50, key is test_row_0/B:col10/1732664131199/Put/seqid=0 2024-11-26T23:35:32,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742332_1508 (size=12301) 2024-11-26T23:35:32,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39750 deadline: 1732664192141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732664192141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732664192142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,150 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 88d8390f4cce6ec80971092bc0ed5e72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,150 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in 88d8390f4cce6ec80971092bc0ed5e72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,150 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_0' in 88d8390f4cce6ec80971092bc0ed5e72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39794 deadline: 1732664192152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,162 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in 88d8390f4cce6ec80971092bc0ed5e72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664192161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,210 INFO [Thread-1885 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1885. 2024-11-26T23:35:32,210 DEBUG [Thread-1885 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:32,210 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,210 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:32,211 INFO [Thread-1881 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1881. 2024-11-26T23:35:32,211 DEBUG [Thread-1881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:32,212 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,212 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:32,215 INFO [Thread-1883 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1883. 2024-11-26T23:35:32,215 DEBUG [Thread-1883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:32,215 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,215 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:32,220 INFO [Thread-1887 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1887. 2024-11-26T23:35:32,220 DEBUG [Thread-1887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:32,220 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,220 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:32,221 INFO [Thread-1889 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1889. 2024-11-26T23:35:32,221 DEBUG [Thread-1889 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:32,221 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,221 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:32,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664192382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/24a26b4f022e4d5ea10ee1659431ca16 2024-11-26T23:35:32,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/e43b1591f6644187b1c9ac92b47fcdff is 50, key is test_row_0/C:col10/1732664131199/Put/seqid=0 2024-11-26T23:35:32,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742333_1509 (size=12301) 2024-11-26T23:35:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T23:35:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39814 deadline: 1732664192691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 , the old value is region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., hostname=c707e9ace675,43077,1732663986534, seqNum=2 because the exception is null or not the one we care about 2024-11-26T23:35:32,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/e43b1591f6644187b1c9ac92b47fcdff 2024-11-26T23:35:32,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/60a7328ebb4942139a65f698c2be6304 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/60a7328ebb4942139a65f698c2be6304 2024-11-26T23:35:32,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/60a7328ebb4942139a65f698c2be6304, entries=150, sequenceid=447, filesize=12.0 K 2024-11-26T23:35:32,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/24a26b4f022e4d5ea10ee1659431ca16 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/24a26b4f022e4d5ea10ee1659431ca16 2024-11-26T23:35:32,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/24a26b4f022e4d5ea10ee1659431ca16, entries=150, sequenceid=447, filesize=12.0 K 2024-11-26T23:35:32,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/e43b1591f6644187b1c9ac92b47fcdff as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/e43b1591f6644187b1c9ac92b47fcdff 2024-11-26T23:35:32,882 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/e43b1591f6644187b1c9ac92b47fcdff, entries=150, sequenceid=447, filesize=12.0 K 2024-11-26T23:35:32,883 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 88d8390f4cce6ec80971092bc0ed5e72 in 1245ms, sequenceid=447, compaction requested=true 2024-11-26T23:35:32,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:32,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:32,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-26T23:35:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-26T23:35:32,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-26T23:35:32,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4690 sec 2024-11-26T23:35:32,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.4730 sec 2024-11-26T23:35:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:33,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-26T23:35:33,203 INFO [Thread-1876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1876. 2024-11-26T23:35:33,203 DEBUG [Thread-1876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:33,203 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:33,203 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:33,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:33,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/5fc3d1b7bf5d43eb81406f1e168ab035 is 50, key is test_row_0/A:col10/1732664133202/Put/seqid=0 2024-11-26T23:35:33,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742334_1510 (size=12301) 2024-11-26T23:35:33,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/5fc3d1b7bf5d43eb81406f1e168ab035 2024-11-26T23:35:33,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/9b8549043f7c4d66afd8b8b42b47d423 is 50, key is test_row_0/B:col10/1732664133202/Put/seqid=0 2024-11-26T23:35:33,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742335_1511 (size=12301) 2024-11-26T23:35:34,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/9b8549043f7c4d66afd8b8b42b47d423 2024-11-26T23:35:34,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/3bb069082a2342a2b3a8eb6498961ef2 is 50, key is test_row_0/C:col10/1732664133202/Put/seqid=0 2024-11-26T23:35:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742336_1512 (size=12301) 2024-11-26T23:35:34,163 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-26T23:35:34,163 INFO [Thread-1872 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1872. 2024-11-26T23:35:34,163 DEBUG [Thread-1872 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:34,163 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:34,163 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,163 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,163 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,163 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:34,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:34,172 INFO [Thread-1874 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1874. 2024-11-26T23:35:34,172 DEBUG [Thread-1874 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:34,172 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,172 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:34,182 INFO [Thread-1878 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1878. 2024-11-26T23:35:34,182 DEBUG [Thread-1878 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:34,182 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,182 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:34,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/3bb069082a2342a2b3a8eb6498961ef2 2024-11-26T23:35:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/5fc3d1b7bf5d43eb81406f1e168ab035 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/5fc3d1b7bf5d43eb81406f1e168ab035 2024-11-26T23:35:34,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/5fc3d1b7bf5d43eb81406f1e168ab035, entries=150, sequenceid=475, filesize=12.0 K 2024-11-26T23:35:34,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/9b8549043f7c4d66afd8b8b42b47d423 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/9b8549043f7c4d66afd8b8b42b47d423 2024-11-26T23:35:34,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/9b8549043f7c4d66afd8b8b42b47d423, entries=150, sequenceid=475, filesize=12.0 K 2024-11-26T23:35:34,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/3bb069082a2342a2b3a8eb6498961ef2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/3bb069082a2342a2b3a8eb6498961ef2 2024-11-26T23:35:34,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/3bb069082a2342a2b3a8eb6498961ef2, entries=150, sequenceid=475, filesize=12.0 K 2024-11-26T23:35:34,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=26.84 KB/27480 for 88d8390f4cce6ec80971092bc0ed5e72 in 1240ms, sequenceid=475, compaction requested=true 2024-11-26T23:35:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:34,443 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88d8390f4cce6ec80971092bc0ed5e72:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:34,443 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:34,444 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:34,444 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:34,444 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/A is initiating minor compaction (all files) 2024-11-26T23:35:34,444 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): 88d8390f4cce6ec80971092bc0ed5e72/B is initiating minor compaction (all files) 2024-11-26T23:35:34,444 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/B in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:34,444 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88d8390f4cce6ec80971092bc0ed5e72/A in TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:34,444 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6d0444b4fec54009a9fae3485c0d517b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b3a0338654d643e58f50cf3884540e7e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/24a26b4f022e4d5ea10ee1659431ca16, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/9b8549043f7c4d66afd8b8b42b47d423] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=49.0 K 2024-11-26T23:35:34,444 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f169d461f3c142278567367ba1efc149, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/73f86b5282db4f70882b45b2e95edd15, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/60a7328ebb4942139a65f698c2be6304, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/5fc3d1b7bf5d43eb81406f1e168ab035] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp, totalSize=51.4 K 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0444b4fec54009a9fae3485c0d517b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732664129249 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting f169d461f3c142278567367ba1efc149, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732664129249 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73f86b5282db4f70882b45b2e95edd15, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732664129932 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a0338654d643e58f50cf3884540e7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732664129953 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a26b4f022e4d5ea10ee1659431ca16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732664131187 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60a7328ebb4942139a65f698c2be6304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732664131187 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b8549043f7c4d66afd8b8b42b47d423, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732664131977 2024-11-26T23:35:34,445 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fc3d1b7bf5d43eb81406f1e168ab035, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732664131977 2024-11-26T23:35:34,452 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:34,452 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88d8390f4cce6ec80971092bc0ed5e72#A#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:34,453 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0659b7f1a2734e6380314c28fb97cb88 is 50, key is test_row_0/B:col10/1732664133202/Put/seqid=0 2024-11-26T23:35:34,453 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/890ede54e9284ad788b6e0f23a43a0fc is 50, key is test_row_0/A:col10/1732664133202/Put/seqid=0 2024-11-26T23:35:34,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742337_1513 (size=13425) 2024-11-26T23:35:34,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742338_1514 (size=13425) 2024-11-26T23:35:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T23:35:34,582 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1269 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3806 rows 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1265 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3793 rows 2024-11-26T23:35:34,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1268 2024-11-26T23:35:34,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3803 rows 2024-11-26T23:35:34,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1263 2024-11-26T23:35:34,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3785 rows 2024-11-26T23:35:34,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1269 2024-11-26T23:35:34,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3802 rows 2024-11-26T23:35:34,583 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:35:34,583 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:35:34,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:34,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:34,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T23:35:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T23:35:34,586 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664134586"}]},"ts":"1732664134586"} 2024-11-26T23:35:34,587 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T23:35:34,587 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T23:35:34,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:35:34,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, UNASSIGN}] 2024-11-26T23:35:34,590 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, UNASSIGN 2024-11-26T23:35:34,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=88d8390f4cce6ec80971092bc0ed5e72, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:34,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, UNASSIGN because future has completed 2024-11-26T23:35:34,592 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:35:34,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:35:34,597 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:35:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T23:35:34,743 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:34,743 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:35:34,744 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 88d8390f4cce6ec80971092bc0ed5e72, disabling compactions & flushes 2024-11-26T23:35:34,744 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:34,862 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/890ede54e9284ad788b6e0f23a43a0fc as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/890ede54e9284ad788b6e0f23a43a0fc 2024-11-26T23:35:34,866 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/A of 88d8390f4cce6ec80971092bc0ed5e72 into 890ede54e9284ad788b6e0f23a43a0fc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:34,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:34,866 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/A, priority=12, startTime=1732664134443; duration=0sec 2024-11-26T23:35:34,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:34,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:A 2024-11-26T23:35:34,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. because compaction request was cancelled 2024-11-26T23:35:34,866 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:C 2024-11-26T23:35:34,867 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/0659b7f1a2734e6380314c28fb97cb88 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0659b7f1a2734e6380314c28fb97cb88 2024-11-26T23:35:34,870 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88d8390f4cce6ec80971092bc0ed5e72/B of 88d8390f4cce6ec80971092bc0ed5e72 into 0659b7f1a2734e6380314c28fb97cb88(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:34,870 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88d8390f4cce6ec80971092bc0ed5e72: 2024-11-26T23:35:34,870 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., storeName=88d8390f4cce6ec80971092bc0ed5e72/B, priority=12, startTime=1732664134443; duration=0sec 2024-11-26T23:35:34,870 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:34,870 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:34,870 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88d8390f4cce6ec80971092bc0ed5e72:B 2024-11-26T23:35:34,870 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:34,870 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. after waiting 0 ms 2024-11-26T23:35:34,870 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:34,870 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 88d8390f4cce6ec80971092bc0ed5e72 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-26T23:35:34,874 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/f40751237d144a8d8e27dcece172eefa is 50, key is test_row_0/A:col10/1732664134162/Put/seqid=0 2024-11-26T23:35:34,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742339_1515 (size=12301) 2024-11-26T23:35:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T23:35:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T23:35:35,277 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/f40751237d144a8d8e27dcece172eefa 2024-11-26T23:35:35,283 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/8a38342a5f884f60a715d896949708c6 is 50, key is test_row_0/B:col10/1732664134162/Put/seqid=0 2024-11-26T23:35:35,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742340_1516 (size=12301) 2024-11-26T23:35:35,687 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/8a38342a5f884f60a715d896949708c6 2024-11-26T23:35:35,692 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/72f0b766128d44f4b78ff21715625316 is 50, key is test_row_0/C:col10/1732664134162/Put/seqid=0 2024-11-26T23:35:35,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742341_1517 (size=12301) 2024-11-26T23:35:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T23:35:36,095 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/72f0b766128d44f4b78ff21715625316 2024-11-26T23:35:36,098 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/A/f40751237d144a8d8e27dcece172eefa as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f40751237d144a8d8e27dcece172eefa 2024-11-26T23:35:36,101 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f40751237d144a8d8e27dcece172eefa, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T23:35:36,101 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/B/8a38342a5f884f60a715d896949708c6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/8a38342a5f884f60a715d896949708c6 2024-11-26T23:35:36,104 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/8a38342a5f884f60a715d896949708c6, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T23:35:36,105 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/.tmp/C/72f0b766128d44f4b78ff21715625316 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/72f0b766128d44f4b78ff21715625316 2024-11-26T23:35:36,107 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/72f0b766128d44f4b78ff21715625316, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T23:35:36,108 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 88d8390f4cce6ec80971092bc0ed5e72 in 1238ms, sequenceid=484, compaction requested=true 2024-11-26T23:35:36,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/0e0c08b555c3420b81c8e0ee9d64feb4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/066b5004788f470ab457c5da9b7e070f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b1fb9740eabc4ae9b4adfa2b3ef5958c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/249b1b3aaf39428d801e533fffaf95eb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c59c49bfd9c1441f953ae00abed18872, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/a9f659c8fcd8464d824506f7b0b12463, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/170dd17ad3c44a4aaeb982ab883291dc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c61b76f81cb34f6799a0aafe5d8ecb24, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/61ed81c660c0497cb634b78ab0b68731, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7d11d24e16204385a9308584595c7f8d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d50d9a80539c47c6b5cf111445770848, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/fb1414a06f724bd39dd198a129e7b260, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7ebd55fc00f84d4c8ebdbf100474d177, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/743d1d6a7a5e4da7a908a6fa73831e05, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/043b67c9e0e848c2914c0a3f10101de7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/157bcc315e8e469cbea6aa7e24591d5d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/14c9ee6387714cb1a70f7b19dd389c54, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/98cec3537461456a97bcb37e78018d5c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/2108cc83bbe349c3b0ca580305ff210f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/794b1a0103d747e0a58d9e9969831223, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b0fe8e5ca6bd4327b6c6db4956a75743, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/82196dfd53a14790bfcd4c15ad431a1f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7fdbb076e8064022acff91c6add487ac, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/dd4b8aeb37b2451a8aa00f9b4b93c7b1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58685bffa86e458bb7839788c4235b06, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58115b9822df4843a1ae6e5f9c9432fc, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/80303ebb26ef4bd6b6fb72f337b1a301, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/8e293232a3404ea3958847306b1a10f8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d5546e8327ca478eb7ba6f756b8c434a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f169d461f3c142278567367ba1efc149, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/73f86b5282db4f70882b45b2e95edd15, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/60a7328ebb4942139a65f698c2be6304, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/5fc3d1b7bf5d43eb81406f1e168ab035] to archive 2024-11-26T23:35:36,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:35:36,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/0e0c08b555c3420b81c8e0ee9d64feb4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/0e0c08b555c3420b81c8e0ee9d64feb4 2024-11-26T23:35:36,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/066b5004788f470ab457c5da9b7e070f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/066b5004788f470ab457c5da9b7e070f 2024-11-26T23:35:36,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b1fb9740eabc4ae9b4adfa2b3ef5958c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b1fb9740eabc4ae9b4adfa2b3ef5958c 2024-11-26T23:35:36,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/249b1b3aaf39428d801e533fffaf95eb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/249b1b3aaf39428d801e533fffaf95eb 2024-11-26T23:35:36,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c59c49bfd9c1441f953ae00abed18872 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c59c49bfd9c1441f953ae00abed18872 2024-11-26T23:35:36,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/a9f659c8fcd8464d824506f7b0b12463 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/a9f659c8fcd8464d824506f7b0b12463 2024-11-26T23:35:36,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/170dd17ad3c44a4aaeb982ab883291dc to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/170dd17ad3c44a4aaeb982ab883291dc 2024-11-26T23:35:36,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c61b76f81cb34f6799a0aafe5d8ecb24 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/c61b76f81cb34f6799a0aafe5d8ecb24 2024-11-26T23:35:36,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/61ed81c660c0497cb634b78ab0b68731 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/61ed81c660c0497cb634b78ab0b68731 2024-11-26T23:35:36,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7d11d24e16204385a9308584595c7f8d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7d11d24e16204385a9308584595c7f8d 2024-11-26T23:35:36,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d50d9a80539c47c6b5cf111445770848 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d50d9a80539c47c6b5cf111445770848 2024-11-26T23:35:36,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/fb1414a06f724bd39dd198a129e7b260 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/fb1414a06f724bd39dd198a129e7b260 2024-11-26T23:35:36,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7ebd55fc00f84d4c8ebdbf100474d177 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7ebd55fc00f84d4c8ebdbf100474d177 2024-11-26T23:35:36,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/743d1d6a7a5e4da7a908a6fa73831e05 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/743d1d6a7a5e4da7a908a6fa73831e05 2024-11-26T23:35:36,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/043b67c9e0e848c2914c0a3f10101de7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/043b67c9e0e848c2914c0a3f10101de7 2024-11-26T23:35:36,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/157bcc315e8e469cbea6aa7e24591d5d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/157bcc315e8e469cbea6aa7e24591d5d 2024-11-26T23:35:36,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/14c9ee6387714cb1a70f7b19dd389c54 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/14c9ee6387714cb1a70f7b19dd389c54 2024-11-26T23:35:36,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/98cec3537461456a97bcb37e78018d5c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/98cec3537461456a97bcb37e78018d5c 2024-11-26T23:35:36,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/2108cc83bbe349c3b0ca580305ff210f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/2108cc83bbe349c3b0ca580305ff210f 2024-11-26T23:35:36,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/794b1a0103d747e0a58d9e9969831223 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/794b1a0103d747e0a58d9e9969831223 2024-11-26T23:35:36,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b0fe8e5ca6bd4327b6c6db4956a75743 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/b0fe8e5ca6bd4327b6c6db4956a75743 2024-11-26T23:35:36,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/82196dfd53a14790bfcd4c15ad431a1f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/82196dfd53a14790bfcd4c15ad431a1f 2024-11-26T23:35:36,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7fdbb076e8064022acff91c6add487ac to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/7fdbb076e8064022acff91c6add487ac 2024-11-26T23:35:36,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/dd4b8aeb37b2451a8aa00f9b4b93c7b1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/dd4b8aeb37b2451a8aa00f9b4b93c7b1 2024-11-26T23:35:36,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58685bffa86e458bb7839788c4235b06 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58685bffa86e458bb7839788c4235b06 2024-11-26T23:35:36,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58115b9822df4843a1ae6e5f9c9432fc to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/58115b9822df4843a1ae6e5f9c9432fc 2024-11-26T23:35:36,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/80303ebb26ef4bd6b6fb72f337b1a301 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/80303ebb26ef4bd6b6fb72f337b1a301 2024-11-26T23:35:36,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/8e293232a3404ea3958847306b1a10f8 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/8e293232a3404ea3958847306b1a10f8 2024-11-26T23:35:36,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d5546e8327ca478eb7ba6f756b8c434a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/d5546e8327ca478eb7ba6f756b8c434a 2024-11-26T23:35:36,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f169d461f3c142278567367ba1efc149 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f169d461f3c142278567367ba1efc149 2024-11-26T23:35:36,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/73f86b5282db4f70882b45b2e95edd15 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/73f86b5282db4f70882b45b2e95edd15 2024-11-26T23:35:36,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/60a7328ebb4942139a65f698c2be6304 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/60a7328ebb4942139a65f698c2be6304 2024-11-26T23:35:36,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/5fc3d1b7bf5d43eb81406f1e168ab035 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/5fc3d1b7bf5d43eb81406f1e168ab035 2024-11-26T23:35:36,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/1101fc55d1434b82a5dce9ad1ab4d733, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0488102ecd18430695e0320f22fa4a7a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/950548fbc35f42b68115ef1d397bb876, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/02207980f5324d9b8909c91418cf68cd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/4e9c51501709428086c3f18c240f72ac, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/732bd1b523da4248a59409bc46ef665d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/f263f074217d47b79fcfe5f186e653c6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/dfb66b33bd9e4462ad9d653e2314db85, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6c7eedab78a04c7292432942c569b78b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0cec1ef383da4366b07f8458982b36e8, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/3fb6eaaaa9fe4d9ca668b7b44aa70775, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e0c5ec37ecee4db3b3702fad51a8b401, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/67798ab9617e45479627901a0f70d0d2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/d07a2d5356184510a40c4c04add3ad24, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b4a5bf45068b4b619a7dc59e12abda82, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e2744457d1a84c22bbc17551ddaa8709, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/2241dad4c703496a842cedca55394885, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5fcf46b740544b97aada09dc60abc28a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/91cef52f710840369b7817e3096cbfb2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/82083e30f26f4a48b92ad69ebdfd4b68, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0d7ba81520c84b4c8a0ca74514902df9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/a33ae26efd3d4f90931305b1dd04436d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/90f20afe05f34fa8a6ce9644bb0e14fe, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5082927db7394c57bb1f937252748ac5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6fb4d266fa3b45e5b83a5086439c15ee, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/97cba938af284a388f8b6b950da206b3, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/ea26fca4fcfd4fae90aad89418ac1e05, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e5fff50715964531bc94dec3b5cea704, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6d0444b4fec54009a9fae3485c0d517b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/211c13602ef949dd8fb9549d9ae31415, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b3a0338654d643e58f50cf3884540e7e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/24a26b4f022e4d5ea10ee1659431ca16, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/9b8549043f7c4d66afd8b8b42b47d423] to archive 2024-11-26T23:35:36,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:35:36,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/1101fc55d1434b82a5dce9ad1ab4d733 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/1101fc55d1434b82a5dce9ad1ab4d733 2024-11-26T23:35:36,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0488102ecd18430695e0320f22fa4a7a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0488102ecd18430695e0320f22fa4a7a 2024-11-26T23:35:36,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/950548fbc35f42b68115ef1d397bb876 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/950548fbc35f42b68115ef1d397bb876 2024-11-26T23:35:36,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/02207980f5324d9b8909c91418cf68cd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/02207980f5324d9b8909c91418cf68cd 2024-11-26T23:35:36,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/4e9c51501709428086c3f18c240f72ac to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/4e9c51501709428086c3f18c240f72ac 2024-11-26T23:35:36,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/732bd1b523da4248a59409bc46ef665d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/732bd1b523da4248a59409bc46ef665d 2024-11-26T23:35:36,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/f263f074217d47b79fcfe5f186e653c6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/f263f074217d47b79fcfe5f186e653c6 2024-11-26T23:35:36,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/dfb66b33bd9e4462ad9d653e2314db85 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/dfb66b33bd9e4462ad9d653e2314db85 2024-11-26T23:35:36,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6c7eedab78a04c7292432942c569b78b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6c7eedab78a04c7292432942c569b78b 2024-11-26T23:35:36,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0cec1ef383da4366b07f8458982b36e8 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0cec1ef383da4366b07f8458982b36e8 2024-11-26T23:35:36,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/3fb6eaaaa9fe4d9ca668b7b44aa70775 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/3fb6eaaaa9fe4d9ca668b7b44aa70775 2024-11-26T23:35:36,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e0c5ec37ecee4db3b3702fad51a8b401 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e0c5ec37ecee4db3b3702fad51a8b401 2024-11-26T23:35:36,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/67798ab9617e45479627901a0f70d0d2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/67798ab9617e45479627901a0f70d0d2 2024-11-26T23:35:36,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/d07a2d5356184510a40c4c04add3ad24 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/d07a2d5356184510a40c4c04add3ad24 2024-11-26T23:35:36,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b4a5bf45068b4b619a7dc59e12abda82 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b4a5bf45068b4b619a7dc59e12abda82 2024-11-26T23:35:36,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e2744457d1a84c22bbc17551ddaa8709 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e2744457d1a84c22bbc17551ddaa8709 2024-11-26T23:35:36,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/2241dad4c703496a842cedca55394885 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/2241dad4c703496a842cedca55394885 2024-11-26T23:35:36,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5fcf46b740544b97aada09dc60abc28a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5fcf46b740544b97aada09dc60abc28a 2024-11-26T23:35:36,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/91cef52f710840369b7817e3096cbfb2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/91cef52f710840369b7817e3096cbfb2 2024-11-26T23:35:36,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/82083e30f26f4a48b92ad69ebdfd4b68 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/82083e30f26f4a48b92ad69ebdfd4b68 2024-11-26T23:35:36,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0d7ba81520c84b4c8a0ca74514902df9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0d7ba81520c84b4c8a0ca74514902df9 2024-11-26T23:35:36,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/a33ae26efd3d4f90931305b1dd04436d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/a33ae26efd3d4f90931305b1dd04436d 2024-11-26T23:35:36,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/90f20afe05f34fa8a6ce9644bb0e14fe to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/90f20afe05f34fa8a6ce9644bb0e14fe 2024-11-26T23:35:36,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5082927db7394c57bb1f937252748ac5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/5082927db7394c57bb1f937252748ac5 2024-11-26T23:35:36,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6fb4d266fa3b45e5b83a5086439c15ee to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6fb4d266fa3b45e5b83a5086439c15ee 2024-11-26T23:35:36,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/97cba938af284a388f8b6b950da206b3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/97cba938af284a388f8b6b950da206b3 2024-11-26T23:35:36,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/ea26fca4fcfd4fae90aad89418ac1e05 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/ea26fca4fcfd4fae90aad89418ac1e05 2024-11-26T23:35:36,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e5fff50715964531bc94dec3b5cea704 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/e5fff50715964531bc94dec3b5cea704 2024-11-26T23:35:36,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6d0444b4fec54009a9fae3485c0d517b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/6d0444b4fec54009a9fae3485c0d517b 2024-11-26T23:35:36,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/211c13602ef949dd8fb9549d9ae31415 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/211c13602ef949dd8fb9549d9ae31415 2024-11-26T23:35:36,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b3a0338654d643e58f50cf3884540e7e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/b3a0338654d643e58f50cf3884540e7e 2024-11-26T23:35:36,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/24a26b4f022e4d5ea10ee1659431ca16 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/24a26b4f022e4d5ea10ee1659431ca16 2024-11-26T23:35:36,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/9b8549043f7c4d66afd8b8b42b47d423 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/9b8549043f7c4d66afd8b8b42b47d423 2024-11-26T23:35:36,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/401586fe66f84fd896cb94dc65ccedb1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/568debd8eddf4c499bceb7fb23d30aea, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fc2be232fdbf48bd96cbe05df17ad6d2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/238ea3a27e624d5db0fd7b3b6ee42a2f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f1ca4a93c37a4271860611d76b0e0c1f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/51d19e93a1554a4598a9cf5dcd3541a0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b556a69fa9d24d82b9902c46080b6017, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/bad827cfdb23458c95bcb21085c78fff, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2bd8b710b15d41e9bb100f4a9deaf56c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/55aad944b1e649ad9a31ebf9bf6a2129, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b99e5b80971045bd84020bd216cc4671, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b0e9641399d0470d80819110ae965a69, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/4aced4302a1e43d2bfbf16585bb989ef, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fe81a2350f294b2ba0b094e094409b2d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/d0e3c0b21eb84a308cd25e6fe437561a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/98e8037f92514a42a08db9f06978177e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/450305a54508484691f57bdd4f06da58, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/8b92a36e5a8a49b1afea4c98fa2d4286, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/80bc7b8a23734f78b4d4395ae940db93, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2e3c332666a24b1ab3f26ab5e920df8f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/833fd88e610f4fc6b789293e3ca4904a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b774bb8d8c0b44fd9062bd248895ed0c, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/38d5a308585b421e8b6937456e0d2022, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/29d5544000924b4e81db99f9e6e51f42, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b3e49f19897741f2a63058bdd5e81faf, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b1b4c51e8fc64fe99af4810427564d86, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b20e892a758149e1ad84729406d46e81, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/ba220090f572493b94abc5423b055d3a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/698a7b7b99a344ef8e984216112af812] to archive 2024-11-26T23:35:36,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:35:36,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/401586fe66f84fd896cb94dc65ccedb1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/401586fe66f84fd896cb94dc65ccedb1 2024-11-26T23:35:36,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/568debd8eddf4c499bceb7fb23d30aea to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/568debd8eddf4c499bceb7fb23d30aea 2024-11-26T23:35:36,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fc2be232fdbf48bd96cbe05df17ad6d2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fc2be232fdbf48bd96cbe05df17ad6d2 2024-11-26T23:35:36,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/238ea3a27e624d5db0fd7b3b6ee42a2f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/238ea3a27e624d5db0fd7b3b6ee42a2f 2024-11-26T23:35:36,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f1ca4a93c37a4271860611d76b0e0c1f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f1ca4a93c37a4271860611d76b0e0c1f 2024-11-26T23:35:36,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/51d19e93a1554a4598a9cf5dcd3541a0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/51d19e93a1554a4598a9cf5dcd3541a0 2024-11-26T23:35:36,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b556a69fa9d24d82b9902c46080b6017 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b556a69fa9d24d82b9902c46080b6017 2024-11-26T23:35:36,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/bad827cfdb23458c95bcb21085c78fff to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/bad827cfdb23458c95bcb21085c78fff 2024-11-26T23:35:36,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2bd8b710b15d41e9bb100f4a9deaf56c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2bd8b710b15d41e9bb100f4a9deaf56c 2024-11-26T23:35:36,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/55aad944b1e649ad9a31ebf9bf6a2129 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/55aad944b1e649ad9a31ebf9bf6a2129 2024-11-26T23:35:36,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b99e5b80971045bd84020bd216cc4671 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b99e5b80971045bd84020bd216cc4671 2024-11-26T23:35:36,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b0e9641399d0470d80819110ae965a69 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b0e9641399d0470d80819110ae965a69 2024-11-26T23:35:36,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/4aced4302a1e43d2bfbf16585bb989ef to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/4aced4302a1e43d2bfbf16585bb989ef 2024-11-26T23:35:36,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fe81a2350f294b2ba0b094e094409b2d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/fe81a2350f294b2ba0b094e094409b2d 2024-11-26T23:35:36,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/d0e3c0b21eb84a308cd25e6fe437561a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/d0e3c0b21eb84a308cd25e6fe437561a 2024-11-26T23:35:36,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/98e8037f92514a42a08db9f06978177e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/98e8037f92514a42a08db9f06978177e 2024-11-26T23:35:36,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/450305a54508484691f57bdd4f06da58 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/450305a54508484691f57bdd4f06da58 2024-11-26T23:35:36,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/8b92a36e5a8a49b1afea4c98fa2d4286 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/8b92a36e5a8a49b1afea4c98fa2d4286 2024-11-26T23:35:36,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/80bc7b8a23734f78b4d4395ae940db93 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/80bc7b8a23734f78b4d4395ae940db93 2024-11-26T23:35:36,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2e3c332666a24b1ab3f26ab5e920df8f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/2e3c332666a24b1ab3f26ab5e920df8f 2024-11-26T23:35:36,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/833fd88e610f4fc6b789293e3ca4904a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/833fd88e610f4fc6b789293e3ca4904a 2024-11-26T23:35:36,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b774bb8d8c0b44fd9062bd248895ed0c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b774bb8d8c0b44fd9062bd248895ed0c 2024-11-26T23:35:36,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/38d5a308585b421e8b6937456e0d2022 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/38d5a308585b421e8b6937456e0d2022 2024-11-26T23:35:36,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/29d5544000924b4e81db99f9e6e51f42 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/29d5544000924b4e81db99f9e6e51f42 2024-11-26T23:35:36,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b3e49f19897741f2a63058bdd5e81faf to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b3e49f19897741f2a63058bdd5e81faf 2024-11-26T23:35:36,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b1b4c51e8fc64fe99af4810427564d86 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b1b4c51e8fc64fe99af4810427564d86 2024-11-26T23:35:36,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b20e892a758149e1ad84729406d46e81 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/b20e892a758149e1ad84729406d46e81 2024-11-26T23:35:36,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/ba220090f572493b94abc5423b055d3a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/ba220090f572493b94abc5423b055d3a 2024-11-26T23:35:36,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/698a7b7b99a344ef8e984216112af812 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/698a7b7b99a344ef8e984216112af812 2024-11-26T23:35:36,190 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/recovered.edits/487.seqid, newMaxSeqId=487, maxSeqId=1 2024-11-26T23:35:36,191 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. 2024-11-26T23:35:36,191 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 88d8390f4cce6ec80971092bc0ed5e72: Waiting for close lock at 1732664134744Running coprocessor pre-close hooks at 1732664134744Disabling compacts and flushes for region at 1732664134744Disabling writes for close at 1732664134870 (+126 ms)Obtaining lock to block concurrent updates at 1732664134870Preparing flush snapshotting stores in 88d8390f4cce6ec80971092bc0ed5e72 at 1732664134870Finished memstore snapshotting TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732664134870Flushing stores of TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72. at 1732664134871 (+1 ms)Flushing 88d8390f4cce6ec80971092bc0ed5e72/A: creating writer at 1732664134871Flushing 88d8390f4cce6ec80971092bc0ed5e72/A: appending metadata at 1732664134873 (+2 ms)Flushing 88d8390f4cce6ec80971092bc0ed5e72/A: closing flushed file at 1732664134873Flushing 88d8390f4cce6ec80971092bc0ed5e72/B: creating writer at 1732664135281 (+408 ms)Flushing 88d8390f4cce6ec80971092bc0ed5e72/B: appending metadata at 1732664135283 (+2 ms)Flushing 88d8390f4cce6ec80971092bc0ed5e72/B: closing flushed file at 1732664135283Flushing 88d8390f4cce6ec80971092bc0ed5e72/C: creating writer at 1732664135689 (+406 ms)Flushing 88d8390f4cce6ec80971092bc0ed5e72/C: appending metadata at 1732664135691 (+2 ms)Flushing 88d8390f4cce6ec80971092bc0ed5e72/C: closing flushed file at 1732664135691Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66568542: reopening flushed file at 1732664136098 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40c82e91: reopening flushed file at 1732664136101 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f401561: reopening flushed file at 1732664136104 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 88d8390f4cce6ec80971092bc0ed5e72 in 1238ms, sequenceid=484, compaction requested=true at 1732664136108 (+4 ms)Writing region close event to WAL at 1732664136188 (+80 ms)Running coprocessor post-close hooks at 1732664136191 (+3 ms)Closed at 1732664136191 2024-11-26T23:35:36,192 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:36,193 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=88d8390f4cce6ec80971092bc0ed5e72, regionState=CLOSED 2024-11-26T23:35:36,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:35:36,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-26T23:35:36,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 88d8390f4cce6ec80971092bc0ed5e72, server=c707e9ace675,43077,1732663986534 in 1.6020 sec 2024-11-26T23:35:36,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-26T23:35:36,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88d8390f4cce6ec80971092bc0ed5e72, UNASSIGN in 1.6060 sec 2024-11-26T23:35:36,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-26T23:35:36,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6090 sec 2024-11-26T23:35:36,200 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664136200"}]},"ts":"1732664136200"} 2024-11-26T23:35:36,201 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T23:35:36,201 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T23:35:36,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6170 sec 2024-11-26T23:35:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T23:35:36,741 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:36,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T23:35:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:36,743 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-26T23:35:36,744 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:36,746 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:36,747 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/recovered.edits] 2024-11-26T23:35:36,749 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/890ede54e9284ad788b6e0f23a43a0fc to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/890ede54e9284ad788b6e0f23a43a0fc 2024-11-26T23:35:36,749 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f40751237d144a8d8e27dcece172eefa to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/A/f40751237d144a8d8e27dcece172eefa 2024-11-26T23:35:36,751 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0659b7f1a2734e6380314c28fb97cb88 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/0659b7f1a2734e6380314c28fb97cb88 2024-11-26T23:35:36,752 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/8a38342a5f884f60a715d896949708c6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/B/8a38342a5f884f60a715d896949708c6 2024-11-26T23:35:36,753 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/3bb069082a2342a2b3a8eb6498961ef2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/3bb069082a2342a2b3a8eb6498961ef2 2024-11-26T23:35:36,754 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/72f0b766128d44f4b78ff21715625316 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/72f0b766128d44f4b78ff21715625316 2024-11-26T23:35:36,754 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/7ebedfe18ba247cb90dc846ae11f089c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/7ebedfe18ba247cb90dc846ae11f089c 2024-11-26T23:35:36,755 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/e43b1591f6644187b1c9ac92b47fcdff to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/e43b1591f6644187b1c9ac92b47fcdff 2024-11-26T23:35:36,756 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f5d0ca4c302d40219700748f87bf4166 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/C/f5d0ca4c302d40219700748f87bf4166 2024-11-26T23:35:36,758 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/recovered.edits/487.seqid to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72/recovered.edits/487.seqid 2024-11-26T23:35:36,759 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/88d8390f4cce6ec80971092bc0ed5e72 2024-11-26T23:35:36,759 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T23:35:36,761 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:36,762 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T23:35:36,764 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T23:35:36,764 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:36,765 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T23:35:36,765 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732664136765"}]},"ts":"9223372036854775807"} 2024-11-26T23:35:36,766 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T23:35:36,766 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 88d8390f4cce6ec80971092bc0ed5e72, NAME => 'TestAcidGuarantees,,1732664110021.88d8390f4cce6ec80971092bc0ed5e72.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T23:35:36,766 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T23:35:36,766 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732664136766"}]},"ts":"9223372036854775807"} 2024-11-26T23:35:36,768 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T23:35:36,768 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:36,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-26T23:35:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-26T23:35:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T23:35:36,852 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:36,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242 (was 238) - Thread LEAK? -, OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 321) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6519 (was 6454) - AvailableMemoryMB LEAK? - 2024-11-26T23:35:36,869 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=6519 2024-11-26T23:35:36,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:35:36,870 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:35:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:36,872 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:35:36,872 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:36,872 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-26T23:35:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T23:35:36,873 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:35:36,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742342_1518 (size=959) 2024-11-26T23:35:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T23:35:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T23:35:37,279 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd 2024-11-26T23:35:37,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742343_1519 (size=53) 2024-11-26T23:35:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T23:35:37,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:35:37,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b63a20c833469cedd4c0bebf55ce9018, disabling compactions & flushes 2024-11-26T23:35:37,684 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:37,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:37,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. after waiting 0 ms 2024-11-26T23:35:37,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:37,684 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:37,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b63a20c833469cedd4c0bebf55ce9018: Waiting for close lock at 1732664137684Disabling compacts and flushes for region at 1732664137684Disabling writes for close at 1732664137684Writing region close event to WAL at 1732664137684Closed at 1732664137684 2024-11-26T23:35:37,685 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:35:37,685 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732664137685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732664137685"}]},"ts":"1732664137685"} 2024-11-26T23:35:37,686 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:35:37,687 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:35:37,687 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664137687"}]},"ts":"1732664137687"} 2024-11-26T23:35:37,688 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T23:35:37,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, ASSIGN}] 2024-11-26T23:35:37,689 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, ASSIGN 2024-11-26T23:35:37,690 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, ASSIGN; state=OFFLINE, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=false 2024-11-26T23:35:37,840 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:37,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, ASSIGN because future has completed 2024-11-26T23:35:37,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:35:37,996 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:37,996 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:35:37,996 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:37,996 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:35:37,996 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:37,996 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:37,997 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:37,998 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b63a20c833469cedd4c0bebf55ce9018 columnFamilyName A 2024-11-26T23:35:37,998 DEBUG [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:37,999 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(327): Store=b63a20c833469cedd4c0bebf55ce9018/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:37,999 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,000 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b63a20c833469cedd4c0bebf55ce9018 columnFamilyName B 2024-11-26T23:35:38,000 DEBUG [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:38,000 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(327): Store=b63a20c833469cedd4c0bebf55ce9018/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:38,000 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,001 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b63a20c833469cedd4c0bebf55ce9018 columnFamilyName C 2024-11-26T23:35:38,001 DEBUG [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:38,001 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(327): Store=b63a20c833469cedd4c0bebf55ce9018/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:38,001 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,002 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,002 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,002 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,002 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,003 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:35:38,004 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,005 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:35:38,005 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened b63a20c833469cedd4c0bebf55ce9018; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67000312, jitterRate=-0.0016175508499145508}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:35:38,005 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:38,006 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for b63a20c833469cedd4c0bebf55ce9018: Running coprocessor pre-open hook at 1732664137996Writing region info on filesystem at 1732664137996Initializing all the Stores at 1732664137997 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664137997Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664137997Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664137997Cleaning up temporary data from old regions at 1732664138002 (+5 ms)Running coprocessor post-open hooks at 1732664138005 (+3 ms)Region opened successfully at 1732664138006 (+1 ms) 2024-11-26T23:35:38,007 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., pid=142, masterSystemTime=1732664137993 2024-11-26T23:35:38,008 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:38,008 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:38,008 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:38,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:35:38,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-26T23:35:38,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 in 168 msec 2024-11-26T23:35:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T23:35:38,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-26T23:35:38,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, ASSIGN in 323 msec 2024-11-26T23:35:38,013 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:35:38,013 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664138013"}]},"ts":"1732664138013"} 2024-11-26T23:35:38,014 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T23:35:38,015 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:35:38,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-26T23:35:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T23:35:39,021 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:39,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2348cc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2136b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,026 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,028 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:35:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35833,1732663985624 2024-11-26T23:35:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7891f70a 2024-11-26T23:35:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:35:39,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:35:39,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T23:35:39,031 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:35:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T23:35:39,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742344_1520 (size=995) 2024-11-26T23:35:39,441 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-26T23:35:39,441 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-26T23:35:39,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:35:39,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, REOPEN/MOVE}] 2024-11-26T23:35:39,445 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, REOPEN/MOVE 2024-11-26T23:35:39,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:39,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, REOPEN/MOVE because future has completed 2024-11-26T23:35:39,447 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:35:39,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:35:39,599 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,599 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:35:39,599 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing b63a20c833469cedd4c0bebf55ce9018, disabling compactions & flushes 2024-11-26T23:35:39,599 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:39,599 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:39,599 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. after waiting 0 ms 2024-11-26T23:35:39,599 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:39,602 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T23:35:39,603 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:39,603 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for b63a20c833469cedd4c0bebf55ce9018: Waiting for close lock at 1732664139599Running coprocessor pre-close hooks at 1732664139599Disabling compacts and flushes for region at 1732664139599Disabling writes for close at 1732664139599Writing region close event to WAL at 1732664139600 (+1 ms)Running coprocessor post-close hooks at 1732664139603 (+3 ms)Closed at 1732664139603 2024-11-26T23:35:39,603 WARN [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: b63a20c833469cedd4c0bebf55ce9018 to self. 2024-11-26T23:35:39,604 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,605 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=CLOSED 2024-11-26T23:35:39,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:35:39,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-26T23:35:39,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 in 159 msec 2024-11-26T23:35:39,608 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, REOPEN/MOVE; state=CLOSED, location=c707e9ace675,43077,1732663986534; forceNewPlan=false, retain=true 2024-11-26T23:35:39,759 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=OPENING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:39,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, REOPEN/MOVE because future has completed 2024-11-26T23:35:39,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:35:39,913 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:39,913 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:35:39,913 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,913 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:35:39,914 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,914 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,915 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,915 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b63a20c833469cedd4c0bebf55ce9018 columnFamilyName A 2024-11-26T23:35:39,916 DEBUG [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:39,917 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(327): Store=b63a20c833469cedd4c0bebf55ce9018/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:39,917 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,917 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b63a20c833469cedd4c0bebf55ce9018 columnFamilyName B 2024-11-26T23:35:39,917 DEBUG [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:39,918 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(327): Store=b63a20c833469cedd4c0bebf55ce9018/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:39,918 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,918 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b63a20c833469cedd4c0bebf55ce9018 columnFamilyName C 2024-11-26T23:35:39,918 DEBUG [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:39,918 INFO [StoreOpener-b63a20c833469cedd4c0bebf55ce9018-1 {}] regionserver.HStore(327): Store=b63a20c833469cedd4c0bebf55ce9018/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:35:39,919 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,919 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,920 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,920 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,920 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,921 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:35:39,922 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,922 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened b63a20c833469cedd4c0bebf55ce9018; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71319067, jitterRate=0.06273691356182098}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:35:39,922 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,923 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for b63a20c833469cedd4c0bebf55ce9018: Running coprocessor pre-open hook at 1732664139914Writing region info on filesystem at 1732664139914Initializing all the Stores at 1732664139914Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664139914Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664139914Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732664139914Cleaning up temporary data from old regions at 1732664139920 (+6 ms)Running coprocessor post-open hooks at 1732664139922 (+2 ms)Region opened successfully at 1732664139923 (+1 ms) 2024-11-26T23:35:39,923 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., pid=147, masterSystemTime=1732664139911 2024-11-26T23:35:39,925 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:39,925 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:39,925 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=OPEN, openSeqNum=5, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:35:39,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:35:39,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-26T23:35:39,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 in 167 msec 2024-11-26T23:35:39,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-26T23:35:39,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, REOPEN/MOVE in 485 msec 2024-11-26T23:35:39,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-26T23:35:39,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-26T23:35:39,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-11-26T23:35:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-26T23:35:39,935 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:39,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,936 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c642ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,937 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24699e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,939 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fade7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,940 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f6bf36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,941 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160f35eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,943 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2175480e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,944 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c757c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,945 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f2571d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,947 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a21543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,948 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b65de67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9b6521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da1658c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d45d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd44bb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,955 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e59b964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a21d968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,958 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bc9e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,959 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fae27c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35833,-1 for getting cluster id 2024-11-26T23:35:39,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:35:39,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ad21b-9223-4523-b167-1f3f7b98c684' 2024-11-26T23:35:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:35:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ad21b-9223-4523-b167-1f3f7b98c684" 2024-11-26T23:35:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c76c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35833,-1] 2024-11-26T23:35:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:35:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:39,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:35:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8c3767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:35:39,962 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,962 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,962 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,962 DEBUG [Thread-2332 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,963 DEBUG [Thread-2328 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,963 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,963 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,963 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,964 DEBUG [Thread-2336 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,964 DEBUG [Thread-2336 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,964 DEBUG [Thread-2330 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,964 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,965 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:35:39,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,43077,1732663986534, seqNum=-1] 2024-11-26T23:35:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:35:39,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,968 DEBUG [Thread-2336 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:35:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-26T23:35:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T23:35:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,971 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5] 2024-11-26T23:35:39,972 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:39,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:39,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:35:40,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bd8ae94fa30d46869e207bf4efe881d0_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664139976/Put/seqid=0 2024-11-26T23:35:40,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742345_1521 (size=12154) 2024-11-26T23:35:40,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664200025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664200026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664200032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664200033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664200033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T23:35:40,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664200142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664200142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664200142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664200142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664200142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:40,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:40,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T23:35:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664200352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664200352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664200352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664200352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664200352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,406 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:40,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bd8ae94fa30d46869e207bf4efe881d0_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bd8ae94fa30d46869e207bf4efe881d0_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:40,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/85c71426bd2f459fbb1a449adc27ee51, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:40,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/85c71426bd2f459fbb1a449adc27ee51 is 175, key is test_row_0/A:col10/1732664139976/Put/seqid=0 2024-11-26T23:35:40,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742346_1522 (size=30955) 2024-11-26T23:35:40,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T23:35:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664200662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664200662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664200662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664200662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664200662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:40,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:40,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:40,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,815 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/85c71426bd2f459fbb1a449adc27ee51 2024-11-26T23:35:40,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/228af8af232f43019da77182d83e719a is 50, key is test_row_0/B:col10/1732664139976/Put/seqid=0 2024-11-26T23:35:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742347_1523 (size=12001) 2024-11-26T23:35:40,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:40,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:40,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:40,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:41,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:41,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T23:35:41,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664201172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:41,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664201172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:41,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664201172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:41,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664201172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:41,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664201172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:41,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:41,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:41,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/228af8af232f43019da77182d83e719a 2024-11-26T23:35:41,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/e2d7b5c268ce43b98abda0e3ef9b2395 is 50, key is test_row_0/C:col10/1732664139976/Put/seqid=0 2024-11-26T23:35:41,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742348_1524 (size=12001) 2024-11-26T23:35:41,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/e2d7b5c268ce43b98abda0e3ef9b2395 2024-11-26T23:35:41,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/85c71426bd2f459fbb1a449adc27ee51 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/85c71426bd2f459fbb1a449adc27ee51 2024-11-26T23:35:41,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/85c71426bd2f459fbb1a449adc27ee51, entries=150, sequenceid=16, filesize=30.2 K 2024-11-26T23:35:41,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/228af8af232f43019da77182d83e719a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/228af8af232f43019da77182d83e719a 2024-11-26T23:35:41,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/228af8af232f43019da77182d83e719a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T23:35:41,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/e2d7b5c268ce43b98abda0e3ef9b2395 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/e2d7b5c268ce43b98abda0e3ef9b2395 2024-11-26T23:35:41,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/e2d7b5c268ce43b98abda0e3ef9b2395, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T23:35:41,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for b63a20c833469cedd4c0bebf55ce9018 in 1304ms, sequenceid=16, compaction requested=false 2024-11-26T23:35:41,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:41,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T23:35:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:41,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T23:35:41,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263d4d53e8c61042019da12dd14e50b995_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664140030/Put/seqid=0 2024-11-26T23:35:41,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742349_1525 (size=12154) 2024-11-26T23:35:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:41,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263d4d53e8c61042019da12dd14e50b995_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263d4d53e8c61042019da12dd14e50b995_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:41,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a3ec23e503a8492785204cd83decd2ee, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:41,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a3ec23e503a8492785204cd83decd2ee is 175, key is test_row_0/A:col10/1732664140030/Put/seqid=0 2024-11-26T23:35:41,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742350_1526 (size=30955) 2024-11-26T23:35:41,708 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:35:41,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a3ec23e503a8492785204cd83decd2ee 2024-11-26T23:35:41,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/445db6b0641846d3b4b992c026f26dbb is 50, key is test_row_0/B:col10/1732664140030/Put/seqid=0 2024-11-26T23:35:41,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742351_1527 (size=12001) 2024-11-26T23:35:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T23:35:42,180 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/445db6b0641846d3b4b992c026f26dbb 2024-11-26T23:35:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:42,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/1faf9480b571484db5b8b15ef8277857 is 50, key is test_row_0/C:col10/1732664140030/Put/seqid=0 2024-11-26T23:35:42,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742352_1528 (size=12001) 2024-11-26T23:35:42,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664202193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664202194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664202195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664202195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664202196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664202301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664202301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664202301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664202302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664202302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664202511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664202511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664202511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664202511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664202511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,591 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/1faf9480b571484db5b8b15ef8277857 2024-11-26T23:35:42,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a3ec23e503a8492785204cd83decd2ee as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a3ec23e503a8492785204cd83decd2ee 2024-11-26T23:35:42,598 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a3ec23e503a8492785204cd83decd2ee, entries=150, sequenceid=42, filesize=30.2 K 2024-11-26T23:35:42,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/445db6b0641846d3b4b992c026f26dbb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/445db6b0641846d3b4b992c026f26dbb 2024-11-26T23:35:42,602 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/445db6b0641846d3b4b992c026f26dbb, entries=150, sequenceid=42, filesize=11.7 K 2024-11-26T23:35:42,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/1faf9480b571484db5b8b15ef8277857 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1faf9480b571484db5b8b15ef8277857 2024-11-26T23:35:42,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1faf9480b571484db5b8b15ef8277857, entries=150, sequenceid=42, filesize=11.7 K 2024-11-26T23:35:42,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for b63a20c833469cedd4c0bebf55ce9018 in 1256ms, sequenceid=42, compaction requested=false 2024-11-26T23:35:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-26T23:35:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-26T23:35:42,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-26T23:35:42,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6360 sec 2024-11-26T23:35:42,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.6480 sec 2024-11-26T23:35:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:42,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:42,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T23:35:42,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267130b78e68ee4924b354780728044e47_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664142824/Put/seqid=0 2024-11-26T23:35:42,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742353_1529 (size=12154) 2024-11-26T23:35:42,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664202848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664202848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664202854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664202855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664202870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664202961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664202961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664202972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664202972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:42,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664202981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664203171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664203171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664203182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664203182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664203191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:43,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267130b78e68ee4924b354780728044e47_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267130b78e68ee4924b354780728044e47_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:43,239 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/5f09e6516edb4e0089dd55f648a995d5, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:43,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/5f09e6516edb4e0089dd55f648a995d5 is 175, key is test_row_0/A:col10/1732664142824/Put/seqid=0 2024-11-26T23:35:43,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742354_1530 (size=30955) 2024-11-26T23:35:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664203481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664203481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664203491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664203492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664203502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,644 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/5f09e6516edb4e0089dd55f648a995d5 2024-11-26T23:35:43,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d is 50, key is test_row_0/B:col10/1732664142824/Put/seqid=0 2024-11-26T23:35:43,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742355_1531 (size=12001) 2024-11-26T23:35:43,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664203991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664203991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:44,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664204002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:44,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664204002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664204011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:44,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d 2024-11-26T23:35:44,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/3506d92e7f2849e4b3ec38241e41614b is 50, key is test_row_0/C:col10/1732664142824/Put/seqid=0 2024-11-26T23:35:44,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742356_1532 (size=12001) 2024-11-26T23:35:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T23:35:44,132 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:44,133 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-26T23:35:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T23:35:44,134 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:44,135 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:44,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T23:35:44,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T23:35:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:44,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T23:35:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T23:35:44,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/3506d92e7f2849e4b3ec38241e41614b 2024-11-26T23:35:44,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/5f09e6516edb4e0089dd55f648a995d5 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/5f09e6516edb4e0089dd55f648a995d5 2024-11-26T23:35:44,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/5f09e6516edb4e0089dd55f648a995d5, entries=150, sequenceid=55, filesize=30.2 K 2024-11-26T23:35:44,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d 2024-11-26T23:35:44,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d, entries=150, sequenceid=55, filesize=11.7 K 2024-11-26T23:35:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/3506d92e7f2849e4b3ec38241e41614b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/3506d92e7f2849e4b3ec38241e41614b 2024-11-26T23:35:44,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/3506d92e7f2849e4b3ec38241e41614b, entries=150, sequenceid=55, filesize=11.7 K 2024-11-26T23:35:44,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for b63a20c833469cedd4c0bebf55ce9018 in 1654ms, sequenceid=55, compaction requested=true 2024-11-26T23:35:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:44,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:44,479 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:44,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:44,479 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:44,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:44,479 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:44,479 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/A is initiating minor compaction (all files) 2024-11-26T23:35:44,479 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/A in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,479 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/85c71426bd2f459fbb1a449adc27ee51, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a3ec23e503a8492785204cd83decd2ee, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/5f09e6516edb4e0089dd55f648a995d5] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=90.7 K 2024-11-26T23:35:44,480 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,480 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/85c71426bd2f459fbb1a449adc27ee51, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a3ec23e503a8492785204cd83decd2ee, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/5f09e6516edb4e0089dd55f648a995d5] 2024-11-26T23:35:44,480 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:44,480 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/B is initiating minor compaction (all files) 2024-11-26T23:35:44,480 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85c71426bd2f459fbb1a449adc27ee51, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732664139974 2024-11-26T23:35:44,480 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/B in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,480 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/228af8af232f43019da77182d83e719a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/445db6b0641846d3b4b992c026f26dbb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.2 K 2024-11-26T23:35:44,480 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ec23e503a8492785204cd83decd2ee, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732664140000 2024-11-26T23:35:44,480 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 228af8af232f43019da77182d83e719a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732664139974 2024-11-26T23:35:44,481 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 445db6b0641846d3b4b992c026f26dbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732664140000 2024-11-26T23:35:44,481 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b3c43ff3d1148ce9dcaeea16ee9ed8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732664142194 2024-11-26T23:35:44,481 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f09e6516edb4e0089dd55f648a995d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732664142194 2024-11-26T23:35:44,486 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:44,487 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:44,488 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/aff46f0570b5442998cf4bb851458b6a is 50, key is test_row_0/B:col10/1732664142824/Put/seqid=0 2024-11-26T23:35:44,489 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126a93df61f65494ae69c1be3284c373656_b63a20c833469cedd4c0bebf55ce9018 store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:44,491 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126a93df61f65494ae69c1be3284c373656_b63a20c833469cedd4c0bebf55ce9018, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:44,491 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a93df61f65494ae69c1be3284c373656_b63a20c833469cedd4c0bebf55ce9018 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:44,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742357_1533 (size=12104) 2024-11-26T23:35:44,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742358_1534 (size=4469) 2024-11-26T23:35:44,495 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#A#compaction#460 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:44,496 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/e151e6ecc11a4b6eb3fa5ea54807ec5a is 175, key is test_row_0/A:col10/1732664142824/Put/seqid=0 2024-11-26T23:35:44,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742359_1535 (size=31058) 2024-11-26T23:35:44,501 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/aff46f0570b5442998cf4bb851458b6a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/aff46f0570b5442998cf4bb851458b6a 2024-11-26T23:35:44,510 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/B of b63a20c833469cedd4c0bebf55ce9018 into aff46f0570b5442998cf4bb851458b6a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:44,510 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:44,510 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/B, priority=13, startTime=1732664144478; duration=0sec 2024-11-26T23:35:44,510 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:44,510 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:B 2024-11-26T23:35:44,510 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:44,511 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:44,511 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/C is initiating minor compaction (all files) 2024-11-26T23:35:44,511 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/C in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,511 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/e2d7b5c268ce43b98abda0e3ef9b2395, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1faf9480b571484db5b8b15ef8277857, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/3506d92e7f2849e4b3ec38241e41614b] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.2 K 2024-11-26T23:35:44,512 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d7b5c268ce43b98abda0e3ef9b2395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732664139974 2024-11-26T23:35:44,512 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 1faf9480b571484db5b8b15ef8277857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732664140000 2024-11-26T23:35:44,512 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 3506d92e7f2849e4b3ec38241e41614b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732664142194 2024-11-26T23:35:44,519 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:44,520 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/99e31eda8b5e4964bff64f862d256139 is 50, key is test_row_0/C:col10/1732664142824/Put/seqid=0 2024-11-26T23:35:44,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742360_1536 (size=12104) 2024-11-26T23:35:44,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T23:35:44,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:44,593 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T23:35:44,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112679c3a8fc42fa477e9ef2ee97715d0921_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664142854/Put/seqid=0 2024-11-26T23:35:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742361_1537 (size=12154) 2024-11-26T23:35:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T23:35:44,905 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/e151e6ecc11a4b6eb3fa5ea54807ec5a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/e151e6ecc11a4b6eb3fa5ea54807ec5a 2024-11-26T23:35:44,909 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/A of b63a20c833469cedd4c0bebf55ce9018 into e151e6ecc11a4b6eb3fa5ea54807ec5a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:44,909 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:44,909 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/A, priority=13, startTime=1732664144478; duration=0sec 2024-11-26T23:35:44,909 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:44,909 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:A 2024-11-26T23:35:44,927 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/99e31eda8b5e4964bff64f862d256139 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/99e31eda8b5e4964bff64f862d256139 2024-11-26T23:35:44,931 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/C of b63a20c833469cedd4c0bebf55ce9018 into 99e31eda8b5e4964bff64f862d256139(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:44,931 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:44,931 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/C, priority=13, startTime=1732664144479; duration=0sec 2024-11-26T23:35:44,931 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:44,931 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:C 2024-11-26T23:35:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:45,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:45,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:45,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112679c3a8fc42fa477e9ef2ee97715d0921_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112679c3a8fc42fa477e9ef2ee97715d0921_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b2f7626054174a5a822bb4578d673d49, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b2f7626054174a5a822bb4578d673d49 is 175, key is test_row_0/A:col10/1732664142854/Put/seqid=0 2024-11-26T23:35:45,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742362_1538 (size=30955) 2024-11-26T23:35:45,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b2f7626054174a5a822bb4578d673d49 2024-11-26T23:35:45,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664205018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/84e4b943cc2f4c74a3bec4d098f2a9e2 is 50, key is test_row_0/B:col10/1732664142854/Put/seqid=0 2024-11-26T23:35:45,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664205021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664205021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664205021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664205022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742363_1539 (size=12001) 2024-11-26T23:35:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,026 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/84e4b943cc2f4c74a3bec4d098f2a9e2 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,027 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_0' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/1d2bfc588cbe404fa93abeb6cd76ec2a is 50, key is test_row_0/C:col10/1732664142854/Put/seqid=0 2024-11-26T23:35:45,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742364_1540 (size=12001) 2024-11-26T23:35:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664205131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664205131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664205131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T23:35:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664205341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664205341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664205341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,437 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/1d2bfc588cbe404fa93abeb6cd76ec2a 2024-11-26T23:35:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b2f7626054174a5a822bb4578d673d49 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b2f7626054174a5a822bb4578d673d49 2024-11-26T23:35:45,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b2f7626054174a5a822bb4578d673d49, entries=150, sequenceid=80, filesize=30.2 K 2024-11-26T23:35:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/84e4b943cc2f4c74a3bec4d098f2a9e2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/84e4b943cc2f4c74a3bec4d098f2a9e2 2024-11-26T23:35:45,447 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/84e4b943cc2f4c74a3bec4d098f2a9e2, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T23:35:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/1d2bfc588cbe404fa93abeb6cd76ec2a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1d2bfc588cbe404fa93abeb6cd76ec2a 2024-11-26T23:35:45,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1d2bfc588cbe404fa93abeb6cd76ec2a, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T23:35:45,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for b63a20c833469cedd4c0bebf55ce9018 in 859ms, sequenceid=80, compaction requested=false 2024-11-26T23:35:45,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:45,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:45,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-26T23:35:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-26T23:35:45,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-26T23:35:45,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3190 sec 2024-11-26T23:35:45,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.3230 sec 2024-11-26T23:35:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:45,654 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:45,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-26T23:35:45,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261d698fc3610d4e3ab963bbff480008cb_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664145652/Put/seqid=0 2024-11-26T23:35:45,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742365_1541 (size=12154) 2024-11-26T23:35:45,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664205683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664205686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664205686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664205792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664205801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664205801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664206002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664206012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664206012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,081 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:46,084 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261d698fc3610d4e3ab963bbff480008cb_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261d698fc3610d4e3ab963bbff480008cb_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:46,085 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/723815347eb74232a94403db8ae385bd, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:46,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/723815347eb74232a94403db8ae385bd is 175, key is test_row_0/A:col10/1732664145652/Put/seqid=0 2024-11-26T23:35:46,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742366_1542 (size=30955) 2024-11-26T23:35:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T23:35:46,282 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:46,282 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-26T23:35:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T23:35:46,284 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:46,285 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:46,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664206311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664206321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664206321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T23:35:46,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T23:35:46,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:46,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/723815347eb74232a94403db8ae385bd 2024-11-26T23:35:46,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/ab28bc83f8f34e209a390dd020307912 is 50, key is test_row_0/B:col10/1732664145652/Put/seqid=0 2024-11-26T23:35:46,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742367_1543 (size=12001) 2024-11-26T23:35:46,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T23:35:46,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:46,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T23:35:46,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T23:35:46,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:46,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664206821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664206831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664206831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:46,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T23:35:46,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:46,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:46,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/ab28bc83f8f34e209a390dd020307912 2024-11-26T23:35:46,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/cc4b0da902314a4ca4602391316adf40 is 50, key is test_row_0/C:col10/1732664145652/Put/seqid=0 2024-11-26T23:35:46,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742368_1544 (size=12001) 2024-11-26T23:35:46,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/cc4b0da902314a4ca4602391316adf40 2024-11-26T23:35:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T23:35:46,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/723815347eb74232a94403db8ae385bd as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/723815347eb74232a94403db8ae385bd 2024-11-26T23:35:46,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/723815347eb74232a94403db8ae385bd, entries=150, sequenceid=98, filesize=30.2 K 2024-11-26T23:35:46,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/ab28bc83f8f34e209a390dd020307912 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/ab28bc83f8f34e209a390dd020307912 2024-11-26T23:35:46,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/ab28bc83f8f34e209a390dd020307912, entries=150, sequenceid=98, filesize=11.7 K 2024-11-26T23:35:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/cc4b0da902314a4ca4602391316adf40 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/cc4b0da902314a4ca4602391316adf40 2024-11-26T23:35:46,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/cc4b0da902314a4ca4602391316adf40, entries=150, sequenceid=98, filesize=11.7 K 2024-11-26T23:35:46,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b63a20c833469cedd4c0bebf55ce9018 in 1269ms, sequenceid=98, compaction requested=true 2024-11-26T23:35:46,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:46,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:46,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:46,923 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:46,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:46,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:46,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:46,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:46,923 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:46,924 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:46,924 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/A is initiating minor compaction (all files) 2024-11-26T23:35:46,924 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/A in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,924 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:46,924 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/e151e6ecc11a4b6eb3fa5ea54807ec5a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b2f7626054174a5a822bb4578d673d49, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/723815347eb74232a94403db8ae385bd] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=90.8 K 2024-11-26T23:35:46,924 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,924 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/B is initiating minor compaction (all files) 2024-11-26T23:35:46,924 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/e151e6ecc11a4b6eb3fa5ea54807ec5a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b2f7626054174a5a822bb4578d673d49, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/723815347eb74232a94403db8ae385bd] 2024-11-26T23:35:46,924 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/B in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:46,925 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/aff46f0570b5442998cf4bb851458b6a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/84e4b943cc2f4c74a3bec4d098f2a9e2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/ab28bc83f8f34e209a390dd020307912] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.3 K 2024-11-26T23:35:46,925 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting e151e6ecc11a4b6eb3fa5ea54807ec5a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732664142194 2024-11-26T23:35:46,925 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting aff46f0570b5442998cf4bb851458b6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732664142194 2024-11-26T23:35:46,926 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2f7626054174a5a822bb4578d673d49, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664142847 2024-11-26T23:35:46,926 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e4b943cc2f4c74a3bec4d098f2a9e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664142847 2024-11-26T23:35:46,926 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 723815347eb74232a94403db8ae385bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732664145652 2024-11-26T23:35:46,926 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting ab28bc83f8f34e209a390dd020307912, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732664145652 2024-11-26T23:35:46,931 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:46,933 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#B#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:46,933 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/cf04d6853fe7405a8aa5ba9584176154 is 50, key is test_row_0/B:col10/1732664145652/Put/seqid=0 2024-11-26T23:35:46,941 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112685fe27cd5e43495980a00f8860070ccb_b63a20c833469cedd4c0bebf55ce9018 store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:46,942 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112685fe27cd5e43495980a00f8860070ccb_b63a20c833469cedd4c0bebf55ce9018, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:46,943 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112685fe27cd5e43495980a00f8860070ccb_b63a20c833469cedd4c0bebf55ce9018 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:46,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742369_1545 (size=12207) 2024-11-26T23:35:46,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742370_1546 (size=4469) 2024-11-26T23:35:47,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T23:35:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:47,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-26T23:35:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:47,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112674ff3ff941264a5cb38d812039340bf1_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664145684/Put/seqid=0 2024-11-26T23:35:47,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742371_1547 (size=12154) 2024-11-26T23:35:47,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664207083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664207088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664207191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664207201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,350 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/cf04d6853fe7405a8aa5ba9584176154 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/cf04d6853fe7405a8aa5ba9584176154 2024-11-26T23:35:47,353 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/B of b63a20c833469cedd4c0bebf55ce9018 into cf04d6853fe7405a8aa5ba9584176154(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:47,353 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:47,353 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/B, priority=13, startTime=1732664146923; duration=0sec 2024-11-26T23:35:47,353 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:47,353 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:B 2024-11-26T23:35:47,354 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:47,354 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:47,354 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/C is initiating minor compaction (all files) 2024-11-26T23:35:47,354 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/C in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:47,355 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/99e31eda8b5e4964bff64f862d256139, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1d2bfc588cbe404fa93abeb6cd76ec2a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/cc4b0da902314a4ca4602391316adf40] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.3 K 2024-11-26T23:35:47,355 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e31eda8b5e4964bff64f862d256139, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732664142194 2024-11-26T23:35:47,355 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2bfc588cbe404fa93abeb6cd76ec2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732664142847 2024-11-26T23:35:47,355 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting cc4b0da902314a4ca4602391316adf40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732664145652 2024-11-26T23:35:47,359 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:47,359 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a4717fb575ab4fffaf6f626130d1dd05 is 175, key is test_row_0/A:col10/1732664145652/Put/seqid=0 2024-11-26T23:35:47,361 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:47,361 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/b419b5f0e79c45479d805029e15021c6 is 50, key is test_row_0/C:col10/1732664145652/Put/seqid=0 2024-11-26T23:35:47,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742372_1548 (size=31161) 2024-11-26T23:35:47,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742373_1549 (size=12207) 2024-11-26T23:35:47,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664207401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664207411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T23:35:47,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:47,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112674ff3ff941264a5cb38d812039340bf1_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112674ff3ff941264a5cb38d812039340bf1_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:47,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a01581631306430a903c4ea542d58cfa, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:47,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a01581631306430a903c4ea542d58cfa is 175, key is test_row_0/A:col10/1732664145684/Put/seqid=0 2024-11-26T23:35:47,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742374_1550 (size=30955) 2024-11-26T23:35:47,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664207711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664207722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,768 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a4717fb575ab4fffaf6f626130d1dd05 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a4717fb575ab4fffaf6f626130d1dd05 2024-11-26T23:35:47,773 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/A of b63a20c833469cedd4c0bebf55ce9018 into a4717fb575ab4fffaf6f626130d1dd05(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:47,773 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:47,773 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/A, priority=13, startTime=1732664146923; duration=0sec 2024-11-26T23:35:47,773 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:47,773 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:A 2024-11-26T23:35:47,773 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/b419b5f0e79c45479d805029e15021c6 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b419b5f0e79c45479d805029e15021c6 2024-11-26T23:35:47,777 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/C of b63a20c833469cedd4c0bebf55ce9018 into b419b5f0e79c45479d805029e15021c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:47,777 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:47,777 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/C, priority=13, startTime=1732664146923; duration=0sec 2024-11-26T23:35:47,777 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:47,777 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:C 2024-11-26T23:35:47,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664207842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664207842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,846 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:47,846 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664207851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:47,854 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:47,871 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a01581631306430a903c4ea542d58cfa 2024-11-26T23:35:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/e6daf7a608fd44b5b48dccea21f1ff4a is 50, key is test_row_0/B:col10/1732664145684/Put/seqid=0 2024-11-26T23:35:47,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742375_1551 (size=12001) 2024-11-26T23:35:48,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664208222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:48,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664208231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:48,290 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/e6daf7a608fd44b5b48dccea21f1ff4a 2024-11-26T23:35:48,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/f3496f9e86f0461f92e8e8ae7bc72003 is 50, key is test_row_0/C:col10/1732664145684/Put/seqid=0 2024-11-26T23:35:48,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742376_1552 (size=12001) 2024-11-26T23:35:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T23:35:48,701 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/f3496f9e86f0461f92e8e8ae7bc72003 2024-11-26T23:35:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a01581631306430a903c4ea542d58cfa as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a01581631306430a903c4ea542d58cfa 2024-11-26T23:35:48,707 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a01581631306430a903c4ea542d58cfa, entries=150, sequenceid=118, filesize=30.2 K 2024-11-26T23:35:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/e6daf7a608fd44b5b48dccea21f1ff4a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/e6daf7a608fd44b5b48dccea21f1ff4a 2024-11-26T23:35:48,711 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/e6daf7a608fd44b5b48dccea21f1ff4a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T23:35:48,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/f3496f9e86f0461f92e8e8ae7bc72003 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/f3496f9e86f0461f92e8e8ae7bc72003 2024-11-26T23:35:48,715 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/f3496f9e86f0461f92e8e8ae7bc72003, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T23:35:48,716 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b63a20c833469cedd4c0bebf55ce9018 in 1668ms, sequenceid=118, compaction requested=false 2024-11-26T23:35:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-26T23:35:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-26T23:35:48,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-26T23:35:48,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4320 sec 2024-11-26T23:35:48,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.4370 sec 2024-11-26T23:35:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:49,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-26T23:35:49,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a1514b466db44b01a4dfefd92d809ca7_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664149242/Put/seqid=0 2024-11-26T23:35:49,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742377_1553 (size=12304) 2024-11-26T23:35:49,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664209279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664209283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664209392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664209392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664209601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664209601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:49,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a1514b466db44b01a4dfefd92d809ca7_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a1514b466db44b01a4dfefd92d809ca7_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:49,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/c99258baa51c41ee829856e3946dded7, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:49,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/c99258baa51c41ee829856e3946dded7 is 175, key is test_row_0/A:col10/1732664149242/Put/seqid=0 2024-11-26T23:35:49,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742378_1554 (size=31105) 2024-11-26T23:35:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664209861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,865 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664209872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664209872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,874 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,874 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664209911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664209911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:50,062 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/c99258baa51c41ee829856e3946dded7 2024-11-26T23:35:50,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c19752426f804aff8c7172f4461d931a is 50, key is test_row_0/B:col10/1732664149242/Put/seqid=0 2024-11-26T23:35:50,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742379_1555 (size=12151) 2024-11-26T23:35:50,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664210421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:50,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:50,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664210421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T23:35:50,441 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:50,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-26T23:35:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T23:35:50,443 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:50,444 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:50,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:50,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c19752426f804aff8c7172f4461d931a 2024-11-26T23:35:50,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/51f27e83893542d08b19e65e1927c89a is 50, key is test_row_0/C:col10/1732664149242/Put/seqid=0 2024-11-26T23:35:50,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742380_1556 (size=12151) 2024-11-26T23:35:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T23:35:50,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T23:35:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T23:35:50,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:50,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T23:35:50,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/51f27e83893542d08b19e65e1927c89a 2024-11-26T23:35:50,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/c99258baa51c41ee829856e3946dded7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/c99258baa51c41ee829856e3946dded7 2024-11-26T23:35:50,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T23:35:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/c99258baa51c41ee829856e3946dded7, entries=150, sequenceid=138, filesize=30.4 K 2024-11-26T23:35:50,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c19752426f804aff8c7172f4461d931a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c19752426f804aff8c7172f4461d931a 2024-11-26T23:35:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:50,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c19752426f804aff8c7172f4461d931a, entries=150, sequenceid=138, filesize=11.9 K 2024-11-26T23:35:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/51f27e83893542d08b19e65e1927c89a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/51f27e83893542d08b19e65e1927c89a 2024-11-26T23:35:50,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/51f27e83893542d08b19e65e1927c89a, entries=150, sequenceid=138, filesize=11.9 K 2024-11-26T23:35:50,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b63a20c833469cedd4c0bebf55ce9018 in 1666ms, sequenceid=138, compaction requested=true 2024-11-26T23:35:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:50,910 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:50,910 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:50,911 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:50,911 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:50,911 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/B is initiating minor compaction (all files) 2024-11-26T23:35:50,911 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/A is initiating minor compaction (all files) 2024-11-26T23:35:50,911 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/B in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,911 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/A in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,911 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a4717fb575ab4fffaf6f626130d1dd05, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a01581631306430a903c4ea542d58cfa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/c99258baa51c41ee829856e3946dded7] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=91.0 K 2024-11-26T23:35:50,911 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/cf04d6853fe7405a8aa5ba9584176154, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/e6daf7a608fd44b5b48dccea21f1ff4a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c19752426f804aff8c7172f4461d931a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.5 K 2024-11-26T23:35:50,911 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,911 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a4717fb575ab4fffaf6f626130d1dd05, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a01581631306430a903c4ea542d58cfa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/c99258baa51c41ee829856e3946dded7] 2024-11-26T23:35:50,912 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting cf04d6853fe7405a8aa5ba9584176154, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732664145652 2024-11-26T23:35:50,912 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4717fb575ab4fffaf6f626130d1dd05, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732664145652 2024-11-26T23:35:50,912 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting e6daf7a608fd44b5b48dccea21f1ff4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732664145673 2024-11-26T23:35:50,912 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01581631306430a903c4ea542d58cfa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732664145673 2024-11-26T23:35:50,913 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99258baa51c41ee829856e3946dded7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732664147081 2024-11-26T23:35:50,913 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c19752426f804aff8c7172f4461d931a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732664147081 2024-11-26T23:35:50,917 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:50,918 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:50,919 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/91a64d5c96c44ce0aa8e8e24f9b2cde1 is 50, key is test_row_0/B:col10/1732664149242/Put/seqid=0 2024-11-26T23:35:50,919 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261df60d48da274c9380369287460238af_b63a20c833469cedd4c0bebf55ce9018 store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:50,921 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261df60d48da274c9380369287460238af_b63a20c833469cedd4c0bebf55ce9018, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:50,921 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261df60d48da274c9380369287460238af_b63a20c833469cedd4c0bebf55ce9018 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:50,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742381_1557 (size=4469) 2024-11-26T23:35:50,925 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#A#compaction#478 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:50,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742382_1558 (size=12459) 2024-11-26T23:35:50,927 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/0438b97e10464ee89e19aa9ca7ddf0c9 is 175, key is test_row_0/A:col10/1732664149242/Put/seqid=0 2024-11-26T23:35:50,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742383_1559 (size=31413) 2024-11-26T23:35:50,932 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/91a64d5c96c44ce0aa8e8e24f9b2cde1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/91a64d5c96c44ce0aa8e8e24f9b2cde1 2024-11-26T23:35:50,942 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/0438b97e10464ee89e19aa9ca7ddf0c9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/0438b97e10464ee89e19aa9ca7ddf0c9 2024-11-26T23:35:50,942 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/B of b63a20c833469cedd4c0bebf55ce9018 into 91a64d5c96c44ce0aa8e8e24f9b2cde1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:50,942 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:50,942 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/B, priority=13, startTime=1732664150910; duration=0sec 2024-11-26T23:35:50,942 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:50,942 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:B 2024-11-26T23:35:50,942 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:50,943 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:50,944 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/C is initiating minor compaction (all files) 2024-11-26T23:35:50,944 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/C in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:50,944 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b419b5f0e79c45479d805029e15021c6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/f3496f9e86f0461f92e8e8ae7bc72003, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/51f27e83893542d08b19e65e1927c89a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.5 K 2024-11-26T23:35:50,945 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b419b5f0e79c45479d805029e15021c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732664145652 2024-11-26T23:35:50,945 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting f3496f9e86f0461f92e8e8ae7bc72003, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732664145673 2024-11-26T23:35:50,945 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 51f27e83893542d08b19e65e1927c89a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732664147081 2024-11-26T23:35:50,947 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/A of b63a20c833469cedd4c0bebf55ce9018 into 0438b97e10464ee89e19aa9ca7ddf0c9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:50,947 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:50,947 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/A, priority=13, startTime=1732664150910; duration=0sec 2024-11-26T23:35:50,947 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:50,948 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:A 2024-11-26T23:35:50,953 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:50,953 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/20c01adb2f124f68b0f3cc17184405e7 is 50, key is test_row_0/C:col10/1732664149242/Put/seqid=0 2024-11-26T23:35:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742384_1560 (size=12459) 2024-11-26T23:35:50,962 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/20c01adb2f124f68b0f3cc17184405e7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/20c01adb2f124f68b0f3cc17184405e7 2024-11-26T23:35:50,968 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/C of b63a20c833469cedd4c0bebf55ce9018 into 20c01adb2f124f68b0f3cc17184405e7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:50,968 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:50,968 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/C, priority=13, startTime=1732664150910; duration=0sec 2024-11-26T23:35:50,968 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:50,968 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:C 2024-11-26T23:35:51,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T23:35:51,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:51,054 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T23:35:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261ca8d2d3254641fc8795fca5fe6c6f3d_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664149282/Put/seqid=0 2024-11-26T23:35:51,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742385_1561 (size=12304) 2024-11-26T23:35:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T23:35:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:51,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:51,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:51,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261ca8d2d3254641fc8795fca5fe6c6f3d_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261ca8d2d3254641fc8795fca5fe6c6f3d_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/8cdc1031dd544d769ab4688cfd416f92, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/8cdc1031dd544d769ab4688cfd416f92 is 175, key is test_row_0/A:col10/1732664149282/Put/seqid=0 2024-11-26T23:35:51,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664211473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:51,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664211474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742386_1562 (size=31105) 2024-11-26T23:35:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T23:35:51,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664211582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664211582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664211791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664211792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:51,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/8cdc1031dd544d769ab4688cfd416f92 2024-11-26T23:35:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c0390ecd4a1e4fcca7a6466545aedb8b is 50, key is test_row_0/B:col10/1732664149282/Put/seqid=0 2024-11-26T23:35:51,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742387_1563 (size=12151) 2024-11-26T23:35:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664212101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664212101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:52,306 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c0390ecd4a1e4fcca7a6466545aedb8b 2024-11-26T23:35:52,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/b3d2efd5d4b1487c9144d392ffbd447e is 50, key is test_row_0/C:col10/1732664149282/Put/seqid=0 2024-11-26T23:35:52,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742388_1564 (size=12151) 2024-11-26T23:35:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T23:35:52,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664212612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:52,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664212612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:52,717 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/b3d2efd5d4b1487c9144d392ffbd447e 2024-11-26T23:35:52,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/8cdc1031dd544d769ab4688cfd416f92 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/8cdc1031dd544d769ab4688cfd416f92 2024-11-26T23:35:52,723 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/8cdc1031dd544d769ab4688cfd416f92, entries=150, sequenceid=160, filesize=30.4 K 2024-11-26T23:35:52,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c0390ecd4a1e4fcca7a6466545aedb8b as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c0390ecd4a1e4fcca7a6466545aedb8b 2024-11-26T23:35:52,727 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c0390ecd4a1e4fcca7a6466545aedb8b, entries=150, sequenceid=160, filesize=11.9 K 2024-11-26T23:35:52,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/b3d2efd5d4b1487c9144d392ffbd447e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b3d2efd5d4b1487c9144d392ffbd447e 2024-11-26T23:35:52,731 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b3d2efd5d4b1487c9144d392ffbd447e, entries=150, sequenceid=160, filesize=11.9 K 2024-11-26T23:35:52,733 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for b63a20c833469cedd4c0bebf55ce9018 in 1679ms, sequenceid=160, compaction requested=false 2024-11-26T23:35:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:52,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-26T23:35:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-26T23:35:52,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-26T23:35:52,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2910 sec 2024-11-26T23:35:52,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2960 sec 2024-11-26T23:35:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:53,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T23:35:53,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112612766a1da56544c994770a7cab6d1496_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664153624/Put/seqid=0 2024-11-26T23:35:53,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742389_1565 (size=14794) 2024-11-26T23:35:53,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664213658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664213659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664213772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664213772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53078 deadline: 1732664213891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53048 deadline: 1732664213891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,894 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,894 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_1' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53042 deadline: 1732664213901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,904 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c707e9ace675,43077,1732663986534 for 'test_row_2' in b63a20c833469cedd4c0bebf55ce9018 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T23:35:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664213981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664213981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:54,036 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:54,039 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112612766a1da56544c994770a7cab6d1496_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112612766a1da56544c994770a7cab6d1496_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:54,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/6c3eaf48029d4dbba76c947b48e10db4, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:54,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/6c3eaf48029d4dbba76c947b48e10db4 is 175, key is test_row_0/A:col10/1732664153624/Put/seqid=0 2024-11-26T23:35:54,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742390_1566 (size=39749) 2024-11-26T23:35:54,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664214291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:54,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664214292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:54,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/6c3eaf48029d4dbba76c947b48e10db4 2024-11-26T23:35:54,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c34675c15edb40bba769965104b7cbf9 is 50, key is test_row_0/B:col10/1732664153624/Put/seqid=0 2024-11-26T23:35:54,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742391_1567 (size=12151) 2024-11-26T23:35:54,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c34675c15edb40bba769965104b7cbf9 2024-11-26T23:35:54,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/d0ea2f58c1194d6981e7390a159e2f3a is 50, key is test_row_0/C:col10/1732664153624/Put/seqid=0 2024-11-26T23:35:54,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742392_1568 (size=12151) 2024-11-26T23:35:54,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/d0ea2f58c1194d6981e7390a159e2f3a 2024-11-26T23:35:54,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/6c3eaf48029d4dbba76c947b48e10db4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/6c3eaf48029d4dbba76c947b48e10db4 2024-11-26T23:35:54,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/6c3eaf48029d4dbba76c947b48e10db4, entries=200, sequenceid=180, filesize=38.8 K 2024-11-26T23:35:54,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/c34675c15edb40bba769965104b7cbf9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c34675c15edb40bba769965104b7cbf9 2024-11-26T23:35:54,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c34675c15edb40bba769965104b7cbf9, entries=150, sequenceid=180, filesize=11.9 K 2024-11-26T23:35:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/d0ea2f58c1194d6981e7390a159e2f3a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/d0ea2f58c1194d6981e7390a159e2f3a 2024-11-26T23:35:54,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/d0ea2f58c1194d6981e7390a159e2f3a, entries=150, sequenceid=180, filesize=11.9 K 2024-11-26T23:35:54,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b63a20c833469cedd4c0bebf55ce9018 in 856ms, sequenceid=180, compaction requested=true 2024-11-26T23:35:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:54,481 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:54,481 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:54,481 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:54,482 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/A is initiating minor compaction (all files) 2024-11-26T23:35:54,482 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:54,482 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/A in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:54,482 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/B is initiating minor compaction (all files) 2024-11-26T23:35:54,482 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/B in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:54,482 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/0438b97e10464ee89e19aa9ca7ddf0c9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/8cdc1031dd544d769ab4688cfd416f92, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/6c3eaf48029d4dbba76c947b48e10db4] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=99.9 K 2024-11-26T23:35:54,482 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:54,482 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/91a64d5c96c44ce0aa8e8e24f9b2cde1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c0390ecd4a1e4fcca7a6466545aedb8b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c34675c15edb40bba769965104b7cbf9] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.9 K 2024-11-26T23:35:54,482 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/0438b97e10464ee89e19aa9ca7ddf0c9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/8cdc1031dd544d769ab4688cfd416f92, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/6c3eaf48029d4dbba76c947b48e10db4] 2024-11-26T23:35:54,483 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a64d5c96c44ce0aa8e8e24f9b2cde1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732664147081 2024-11-26T23:35:54,483 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c0390ecd4a1e4fcca7a6466545aedb8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664149273 2024-11-26T23:35:54,483 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting c34675c15edb40bba769965104b7cbf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732664151472 2024-11-26T23:35:54,489 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:54,489 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/8f1a2670b6d6461fa188c0deda728497 is 50, key is test_row_0/B:col10/1732664153624/Put/seqid=0 2024-11-26T23:35:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742393_1569 (size=12561) 2024-11-26T23:35:54,496 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0438b97e10464ee89e19aa9ca7ddf0c9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732664147081 2024-11-26T23:35:54,496 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cdc1031dd544d769ab4688cfd416f92, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664149273 2024-11-26T23:35:54,497 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c3eaf48029d4dbba76c947b48e10db4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732664151471 2024-11-26T23:35:54,501 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:54,502 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112601fd583f0dee4052ab45eeccf57e5af6_b63a20c833469cedd4c0bebf55ce9018 store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:54,504 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112601fd583f0dee4052ab45eeccf57e5af6_b63a20c833469cedd4c0bebf55ce9018, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:54,504 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112601fd583f0dee4052ab45eeccf57e5af6_b63a20c833469cedd4c0bebf55ce9018 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:54,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742394_1570 (size=4469) 2024-11-26T23:35:54,508 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#A#compaction#487 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:54,508 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a5af32bab9954f6f96ed36842b4f1646 is 175, key is test_row_0/A:col10/1732664153624/Put/seqid=0 2024-11-26T23:35:54,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742395_1571 (size=31515) 2024-11-26T23:35:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T23:35:54,601 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:54,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-26T23:35:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T23:35:54,604 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:54,605 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:54,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T23:35:54,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T23:35:54,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:54,758 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-26T23:35:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112620e5f7e76a6743479db78b580f24030c_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664153657/Put/seqid=0 2024-11-26T23:35:54,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742396_1572 (size=12304) 2024-11-26T23:35:54,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:54,771 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112620e5f7e76a6743479db78b580f24030c_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112620e5f7e76a6743479db78b580f24030c_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/dc07f6b12a7748f2a9521ceb1898864d, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/dc07f6b12a7748f2a9521ceb1898864d is 175, key is test_row_0/A:col10/1732664153657/Put/seqid=0 2024-11-26T23:35:54,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742397_1573 (size=31105) 2024-11-26T23:35:54,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/dc07f6b12a7748f2a9521ceb1898864d 2024-11-26T23:35:54,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/5b79cfda99d44326b4b5bbd93f972706 is 50, key is test_row_0/B:col10/1732664153657/Put/seqid=0 2024-11-26T23:35:54,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742398_1574 (size=12151) 2024-11-26T23:35:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:54,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:54,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664214840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:54,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:54,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664214843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:54,897 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/8f1a2670b6d6461fa188c0deda728497 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8f1a2670b6d6461fa188c0deda728497 2024-11-26T23:35:54,901 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/B of b63a20c833469cedd4c0bebf55ce9018 into 8f1a2670b6d6461fa188c0deda728497(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:54,901 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:54,901 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/B, priority=13, startTime=1732664154481; duration=0sec 2024-11-26T23:35:54,901 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:54,901 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:B 2024-11-26T23:35:54,901 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:35:54,902 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:35:54,902 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/C is initiating minor compaction (all files) 2024-11-26T23:35:54,902 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/C in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:54,902 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/20c01adb2f124f68b0f3cc17184405e7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b3d2efd5d4b1487c9144d392ffbd447e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/d0ea2f58c1194d6981e7390a159e2f3a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=35.9 K 2024-11-26T23:35:54,902 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c01adb2f124f68b0f3cc17184405e7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732664147081 2024-11-26T23:35:54,902 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d2efd5d4b1487c9144d392ffbd447e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732664149273 2024-11-26T23:35:54,903 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ea2f58c1194d6981e7390a159e2f3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732664151472 2024-11-26T23:35:54,908 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#C#compaction#490 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:54,908 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/71cd1be639164918bdead4088bd628d4 is 50, key is test_row_0/C:col10/1732664153624/Put/seqid=0 2024-11-26T23:35:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742399_1575 (size=12561) 2024-11-26T23:35:54,915 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/a5af32bab9954f6f96ed36842b4f1646 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a5af32bab9954f6f96ed36842b4f1646 2024-11-26T23:35:54,919 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/A of b63a20c833469cedd4c0bebf55ce9018 into a5af32bab9954f6f96ed36842b4f1646(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:54,919 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:54,919 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/A, priority=13, startTime=1732664154481; duration=0sec 2024-11-26T23:35:54,919 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:54,919 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:A 2024-11-26T23:35:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T23:35:54,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664214952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:54,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664214952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664215162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:55,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664215162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:55,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/5b79cfda99d44326b4b5bbd93f972706 2024-11-26T23:35:55,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/10227e9595964402bcac11866c38e224 is 50, key is test_row_0/C:col10/1732664153657/Put/seqid=0 2024-11-26T23:35:55,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742400_1576 (size=12151) 2024-11-26T23:35:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T23:35:55,315 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/71cd1be639164918bdead4088bd628d4 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/71cd1be639164918bdead4088bd628d4 2024-11-26T23:35:55,319 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/C of b63a20c833469cedd4c0bebf55ce9018 into 71cd1be639164918bdead4088bd628d4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:55,319 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:55,319 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/C, priority=13, startTime=1732664154481; duration=0sec 2024-11-26T23:35:55,319 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:55,319 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:C 2024-11-26T23:35:55,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664215471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664215471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:55,597 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/10227e9595964402bcac11866c38e224 2024-11-26T23:35:55,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/dc07f6b12a7748f2a9521ceb1898864d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/dc07f6b12a7748f2a9521ceb1898864d 2024-11-26T23:35:55,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/dc07f6b12a7748f2a9521ceb1898864d, entries=150, sequenceid=196, filesize=30.4 K 2024-11-26T23:35:55,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/5b79cfda99d44326b4b5bbd93f972706 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5b79cfda99d44326b4b5bbd93f972706 2024-11-26T23:35:55,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5b79cfda99d44326b4b5bbd93f972706, entries=150, sequenceid=196, filesize=11.9 K 2024-11-26T23:35:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/10227e9595964402bcac11866c38e224 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/10227e9595964402bcac11866c38e224 2024-11-26T23:35:55,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/10227e9595964402bcac11866c38e224, entries=150, sequenceid=196, filesize=11.9 K 2024-11-26T23:35:55,619 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b63a20c833469cedd4c0bebf55ce9018 in 862ms, sequenceid=196, compaction requested=false 2024-11-26T23:35:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-26T23:35:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-26T23:35:55,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-26T23:35:55,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-26T23:35:55,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0200 sec 2024-11-26T23:35:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T23:35:55,741 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:55,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-26T23:35:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T23:35:55,743 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:55,744 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:55,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T23:35:55,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T23:35:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:55,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-26T23:35:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e37dfedace3743bf9de56c96687f25ff_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664154839/Put/seqid=0 2024-11-26T23:35:55,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742401_1577 (size=12304) 2024-11-26T23:35:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:56,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:56,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664216032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664216037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T23:35:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664216142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664216142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:56,311 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e37dfedace3743bf9de56c96687f25ff_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e37dfedace3743bf9de56c96687f25ff_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:56,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/d00c95034f4c4e6e828461d89731fc2e, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:56,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/d00c95034f4c4e6e828461d89731fc2e is 175, key is test_row_0/A:col10/1732664154839/Put/seqid=0 2024-11-26T23:35:56,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742402_1578 (size=31105) 2024-11-26T23:35:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664216352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664216352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T23:35:56,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664216661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664216661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:56,717 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/d00c95034f4c4e6e828461d89731fc2e 2024-11-26T23:35:56,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/83e6fcd414bd45309caeb201efbbaf57 is 50, key is test_row_0/B:col10/1732664154839/Put/seqid=0 2024-11-26T23:35:56,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742403_1579 (size=12151) 2024-11-26T23:35:56,728 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/83e6fcd414bd45309caeb201efbbaf57 2024-11-26T23:35:56,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/46d8eaddf7114480b12a39d04b6c85c2 is 50, key is test_row_0/C:col10/1732664154839/Put/seqid=0 2024-11-26T23:35:56,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742404_1580 (size=12151) 2024-11-26T23:35:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T23:35:57,141 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/46d8eaddf7114480b12a39d04b6c85c2 2024-11-26T23:35:57,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/d00c95034f4c4e6e828461d89731fc2e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/d00c95034f4c4e6e828461d89731fc2e 2024-11-26T23:35:57,148 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/d00c95034f4c4e6e828461d89731fc2e, entries=150, sequenceid=219, filesize=30.4 K 2024-11-26T23:35:57,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/83e6fcd414bd45309caeb201efbbaf57 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/83e6fcd414bd45309caeb201efbbaf57 2024-11-26T23:35:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/83e6fcd414bd45309caeb201efbbaf57, entries=150, sequenceid=219, filesize=11.9 K 2024-11-26T23:35:57,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/46d8eaddf7114480b12a39d04b6c85c2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/46d8eaddf7114480b12a39d04b6c85c2 2024-11-26T23:35:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,156 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/46d8eaddf7114480b12a39d04b6c85c2, entries=150, sequenceid=219, filesize=11.9 K 2024-11-26T23:35:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b63a20c833469cedd4c0bebf55ce9018 in 1260ms, sequenceid=219, compaction requested=true 2024-11-26T23:35:57,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:57,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:57,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-26T23:35:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-26T23:35:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-26T23:35:57,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-26T23:35:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4190 sec 2024-11-26T23:35:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:57,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:35:57,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-26T23:35:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f7b765e61f14a03a1cf32a119964a57_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664156035/Put/seqid=0 2024-11-26T23:35:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742405_1581 (size=14794) 2024-11-26T23:35:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664217232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664217232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664217342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664217342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664217551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664217551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:57,593 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f7b765e61f14a03a1cf32a119964a57_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f7b765e61f14a03a1cf32a119964a57_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:57,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/255d0a96bdd64a5aa47f9c0e32c941d0, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:57,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/255d0a96bdd64a5aa47f9c0e32c941d0 is 175, key is test_row_0/A:col10/1732664156035/Put/seqid=0 2024-11-26T23:35:57,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742406_1582 (size=39745) 2024-11-26T23:35:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664217862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664217862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T23:35:57,891 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:35:57,892 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T23:35:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-26T23:35:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T23:35:57,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:35:57,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:35:57,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:35:57,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/255d0a96bdd64a5aa47f9c0e32c941d0 2024-11-26T23:35:58,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T23:35:58,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/2fc555cf9fc54b71a7e6f2cd820b00f9 is 50, key is test_row_0/B:col10/1732664156035/Put/seqid=0 2024-11-26T23:35:58,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742407_1583 (size=9757) 2024-11-26T23:35:58,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T23:35:58,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T23:35:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T23:35:58,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T23:35:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664218372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664218372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:58,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/2fc555cf9fc54b71a7e6f2cd820b00f9 2024-11-26T23:35:58,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/528f194acb7e4dfc9025648eb7ed2aeb is 50, key is test_row_0/C:col10/1732664156035/Put/seqid=0 2024-11-26T23:35:58,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742408_1584 (size=9757) 2024-11-26T23:35:58,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T23:35:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T23:35:58,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T23:35:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T23:35:58,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:58,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:35:58,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/528f194acb7e4dfc9025648eb7ed2aeb 2024-11-26T23:35:58,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/255d0a96bdd64a5aa47f9c0e32c941d0 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/255d0a96bdd64a5aa47f9c0e32c941d0 2024-11-26T23:35:58,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/255d0a96bdd64a5aa47f9c0e32c941d0, entries=200, sequenceid=237, filesize=38.8 K 2024-11-26T23:35:58,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/2fc555cf9fc54b71a7e6f2cd820b00f9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2fc555cf9fc54b71a7e6f2cd820b00f9 2024-11-26T23:35:58,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2fc555cf9fc54b71a7e6f2cd820b00f9, entries=100, sequenceid=237, filesize=9.5 K 2024-11-26T23:35:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/528f194acb7e4dfc9025648eb7ed2aeb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/528f194acb7e4dfc9025648eb7ed2aeb 2024-11-26T23:35:58,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/528f194acb7e4dfc9025648eb7ed2aeb, entries=100, sequenceid=237, filesize=9.5 K 2024-11-26T23:35:58,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for b63a20c833469cedd4c0bebf55ce9018 in 1656ms, sequenceid=237, compaction requested=true 2024-11-26T23:35:58,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:58,835 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:58,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:35:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:35:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:35:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:58,836 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:58,836 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133470 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:58,836 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/A is initiating minor compaction (all files) 2024-11-26T23:35:58,836 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/A in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,837 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a5af32bab9954f6f96ed36842b4f1646, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/dc07f6b12a7748f2a9521ceb1898864d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/d00c95034f4c4e6e828461d89731fc2e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/255d0a96bdd64a5aa47f9c0e32c941d0] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=130.3 K 2024-11-26T23:35:58,837 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,837 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:58,837 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a5af32bab9954f6f96ed36842b4f1646, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/dc07f6b12a7748f2a9521ceb1898864d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/d00c95034f4c4e6e828461d89731fc2e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/255d0a96bdd64a5aa47f9c0e32c941d0] 2024-11-26T23:35:58,837 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/B is initiating minor compaction (all files) 2024-11-26T23:35:58,837 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/B in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,837 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8f1a2670b6d6461fa188c0deda728497, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5b79cfda99d44326b4b5bbd93f972706, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/83e6fcd414bd45309caeb201efbbaf57, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2fc555cf9fc54b71a7e6f2cd820b00f9] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=45.5 K 2024-11-26T23:35:58,837 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5af32bab9954f6f96ed36842b4f1646, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732664151472 2024-11-26T23:35:58,837 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc07f6b12a7748f2a9521ceb1898864d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732664153639 2024-11-26T23:35:58,837 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1a2670b6d6461fa188c0deda728497, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732664151472 2024-11-26T23:35:58,837 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting d00c95034f4c4e6e828461d89731fc2e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732664154833 2024-11-26T23:35:58,838 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b79cfda99d44326b4b5bbd93f972706, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732664153639 2024-11-26T23:35:58,838 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 255d0a96bdd64a5aa47f9c0e32c941d0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732664156030 2024-11-26T23:35:58,838 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e6fcd414bd45309caeb201efbbaf57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732664154833 2024-11-26T23:35:58,838 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fc555cf9fc54b71a7e6f2cd820b00f9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732664156035 2024-11-26T23:35:58,843 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:58,845 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:58,845 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261e61f1f4fd4d442581d46729e1c39279_b63a20c833469cedd4c0bebf55ce9018 store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:58,846 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/3f97e4ca016c4b69ab61d65346d1c39a is 50, key is test_row_0/B:col10/1732664156035/Put/seqid=0 2024-11-26T23:35:58,847 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261e61f1f4fd4d442581d46729e1c39279_b63a20c833469cedd4c0bebf55ce9018, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:58,847 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e61f1f4fd4d442581d46729e1c39279_b63a20c833469cedd4c0bebf55ce9018 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:58,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742409_1585 (size=4469) 2024-11-26T23:35:58,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742410_1586 (size=12697) 2024-11-26T23:35:58,867 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/3f97e4ca016c4b69ab61d65346d1c39a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/3f97e4ca016c4b69ab61d65346d1c39a 2024-11-26T23:35:58,872 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/B of b63a20c833469cedd4c0bebf55ce9018 into 3f97e4ca016c4b69ab61d65346d1c39a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:58,872 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:58,872 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/B, priority=12, startTime=1732664158836; duration=0sec 2024-11-26T23:35:58,872 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:35:58,872 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:B 2024-11-26T23:35:58,872 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:35:58,873 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T23:35:58,873 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/C is initiating minor compaction (all files) 2024-11-26T23:35:58,873 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/C in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,873 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/71cd1be639164918bdead4088bd628d4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/10227e9595964402bcac11866c38e224, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/46d8eaddf7114480b12a39d04b6c85c2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/528f194acb7e4dfc9025648eb7ed2aeb] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=45.5 K 2024-11-26T23:35:58,874 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 71cd1be639164918bdead4088bd628d4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732664151472 2024-11-26T23:35:58,874 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 10227e9595964402bcac11866c38e224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732664153639 2024-11-26T23:35:58,874 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d8eaddf7114480b12a39d04b6c85c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732664154833 2024-11-26T23:35:58,875 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 528f194acb7e4dfc9025648eb7ed2aeb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732664156035 2024-11-26T23:35:58,882 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#C#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:58,882 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/357ac4f757384390b46faf26d5599e2f is 50, key is test_row_0/C:col10/1732664156035/Put/seqid=0 2024-11-26T23:35:58,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742411_1587 (size=12697) 2024-11-26T23:35:58,898 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/357ac4f757384390b46faf26d5599e2f as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/357ac4f757384390b46faf26d5599e2f 2024-11-26T23:35:58,903 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/C of b63a20c833469cedd4c0bebf55ce9018 into 357ac4f757384390b46faf26d5599e2f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:58,903 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:58,903 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/C, priority=12, startTime=1732664158836; duration=0sec 2024-11-26T23:35:58,903 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:58,904 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:C 2024-11-26T23:35:58,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43077 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T23:35:58,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:35:58,962 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T23:35:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265c70f6c8791e46b69e67aefe0066beda_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664157197/Put/seqid=0 2024-11-26T23:35:58,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742412_1588 (size=12304) 2024-11-26T23:35:58,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:35:58,974 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265c70f6c8791e46b69e67aefe0066beda_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265c70f6c8791e46b69e67aefe0066beda_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/807d1195a9f644a28ec6df9e44f5c3e2, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:35:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/807d1195a9f644a28ec6df9e44f5c3e2 is 175, key is test_row_0/A:col10/1732664157197/Put/seqid=0 2024-11-26T23:35:58,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742413_1589 (size=31105) 2024-11-26T23:35:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T23:35:59,255 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:35:59,255 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/081f8696ecd04ed5aa573f14696dfd62 is 175, key is test_row_0/A:col10/1732664156035/Put/seqid=0 2024-11-26T23:35:59,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742414_1590 (size=31758) 2024-11-26T23:35:59,379 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/807d1195a9f644a28ec6df9e44f5c3e2 2024-11-26T23:35:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:35:59,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. as already flushing 2024-11-26T23:35:59,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/8a30e8ef23c243c39ce34e6831369f7d is 50, key is test_row_0/B:col10/1732664157197/Put/seqid=0 2024-11-26T23:35:59,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742415_1591 (size=12151) 2024-11-26T23:35:59,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664219416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:59,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664219420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:59,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664219531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664219531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:59,663 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/081f8696ecd04ed5aa573f14696dfd62 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/081f8696ecd04ed5aa573f14696dfd62 2024-11-26T23:35:59,667 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/A of b63a20c833469cedd4c0bebf55ce9018 into 081f8696ecd04ed5aa573f14696dfd62(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:35:59,667 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:35:59,667 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/A, priority=12, startTime=1732664158835; duration=0sec 2024-11-26T23:35:59,667 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:35:59,667 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:A 2024-11-26T23:35:59,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664219742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:59,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:35:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664219742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:35:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:35:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:35:59,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/8a30e8ef23c243c39ce34e6831369f7d 2024-11-26T23:35:59,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/726ba56a1bbc476a8d7b155396b3c716 is 50, key is test_row_0/C:col10/1732664157197/Put/seqid=0 2024-11-26T23:35:59,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742416_1592 (size=12151) 2024-11-26T23:35:59,966 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-11-26T23:35:59,966 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:59,966 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,966 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,966 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-26T23:35:59,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:59,966 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:59,966 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,966 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:59,967 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-26T23:35:59,967 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:59,967 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,967 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:59,969 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-26T23:35:59,969 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:59,969 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,969 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:35:59,969 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-26T23:35:59,969 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:35:59,969 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,970 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:35:59,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T23:36:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:36:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53050 deadline: 1732664220052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:36:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:36:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1732664220052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 2024-11-26T23:36:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:36:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:36:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:36:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 , the old value is region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:36:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:36:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., hostname=c707e9ace675,43077,1732663986534, seqNum=5 because the exception is null or not the one we care about 2024-11-26T23:36:00,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/726ba56a1bbc476a8d7b155396b3c716 2024-11-26T23:36:00,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/807d1195a9f644a28ec6df9e44f5c3e2 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/807d1195a9f644a28ec6df9e44f5c3e2 2024-11-26T23:36:00,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/807d1195a9f644a28ec6df9e44f5c3e2, entries=150, sequenceid=258, filesize=30.4 K 2024-11-26T23:36:00,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/8a30e8ef23c243c39ce34e6831369f7d as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8a30e8ef23c243c39ce34e6831369f7d 2024-11-26T23:36:00,208 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8a30e8ef23c243c39ce34e6831369f7d, entries=150, sequenceid=258, filesize=11.9 K 2024-11-26T23:36:00,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/726ba56a1bbc476a8d7b155396b3c716 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/726ba56a1bbc476a8d7b155396b3c716 2024-11-26T23:36:00,211 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/726ba56a1bbc476a8d7b155396b3c716, entries=150, sequenceid=258, filesize=11.9 K 2024-11-26T23:36:00,211 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b63a20c833469cedd4c0bebf55ce9018 in 1249ms, sequenceid=258, compaction requested=false 2024-11-26T23:36:00,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:36:00,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:00,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-26T23:36:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-26T23:36:00,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-26T23:36:00,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3190 sec 2024-11-26T23:36:00,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.3220 sec 2024-11-26T23:36:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43077 {}] regionserver.HRegion(8855): Flush requested on b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:00,563 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:36:00,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T23:36:00,563 INFO [Thread-2332 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2332. 2024-11-26T23:36:00,563 INFO [Thread-2328 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2328. 2024-11-26T23:36:00,563 DEBUG [Thread-2332 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:36:00,563 DEBUG [Thread-2328 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:36:00,563 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:00,563 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:00,563 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:00,563 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:00,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:00,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:00,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126643f3e433ff64c37b135a81f731be59b_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_0/A:col10/1732664159408/Put/seqid=0 2024-11-26T23:36:00,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742417_1593 (size=12454) 2024-11-26T23:36:00,973 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:36:00,976 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126643f3e433ff64c37b135a81f731be59b_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126643f3e433ff64c37b135a81f731be59b_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:00,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b8512e3c3e484e91be939ac572d08dfb, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:36:00,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b8512e3c3e484e91be939ac572d08dfb is 175, key is test_row_0/A:col10/1732664159408/Put/seqid=0 2024-11-26T23:36:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742418_1594 (size=31255) 2024-11-26T23:36:01,381 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b8512e3c3e484e91be939ac572d08dfb 2024-11-26T23:36:01,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/5ea4a78abdf74d79a70b041fb629d64a is 50, key is test_row_0/B:col10/1732664159408/Put/seqid=0 2024-11-26T23:36:01,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742419_1595 (size=12301) 2024-11-26T23:36:01,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/5ea4a78abdf74d79a70b041fb629d64a 2024-11-26T23:36:01,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/abb5a945400645f5a2b63a589cdf11a3 is 50, key is test_row_0/C:col10/1732664159408/Put/seqid=0 2024-11-26T23:36:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742420_1596 (size=12301) 2024-11-26T23:36:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T23:36:02,072 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:36:02,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/abb5a945400645f5a2b63a589cdf11a3 2024-11-26T23:36:02,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b8512e3c3e484e91be939ac572d08dfb as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b8512e3c3e484e91be939ac572d08dfb 2024-11-26T23:36:02,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b8512e3c3e484e91be939ac572d08dfb, entries=150, sequenceid=278, filesize=30.5 K 2024-11-26T23:36:02,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/5ea4a78abdf74d79a70b041fb629d64a as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5ea4a78abdf74d79a70b041fb629d64a 2024-11-26T23:36:02,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5ea4a78abdf74d79a70b041fb629d64a, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T23:36:02,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/abb5a945400645f5a2b63a589cdf11a3 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/abb5a945400645f5a2b63a589cdf11a3 2024-11-26T23:36:02,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/abb5a945400645f5a2b63a589cdf11a3, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T23:36:02,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for b63a20c833469cedd4c0bebf55ce9018 in 1647ms, sequenceid=278, compaction requested=true 2024-11-26T23:36:02,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:36:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:36:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:36:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:36:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:36:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b63a20c833469cedd4c0bebf55ce9018:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T23:36:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:36:02,211 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:36:02,211 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:36:02,211 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94118 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:36:02,211 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:36:02,211 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/B is initiating minor compaction (all files) 2024-11-26T23:36:02,211 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/A is initiating minor compaction (all files) 2024-11-26T23:36:02,212 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/B in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:02,212 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/A in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:02,212 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/3f97e4ca016c4b69ab61d65346d1c39a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8a30e8ef23c243c39ce34e6831369f7d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5ea4a78abdf74d79a70b041fb629d64a] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=36.3 K 2024-11-26T23:36:02,212 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/081f8696ecd04ed5aa573f14696dfd62, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/807d1195a9f644a28ec6df9e44f5c3e2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b8512e3c3e484e91be939ac572d08dfb] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=91.9 K 2024-11-26T23:36:02,212 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:02,212 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. files: [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/081f8696ecd04ed5aa573f14696dfd62, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/807d1195a9f644a28ec6df9e44f5c3e2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b8512e3c3e484e91be939ac572d08dfb] 2024-11-26T23:36:02,212 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f97e4ca016c4b69ab61d65346d1c39a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732664154842 2024-11-26T23:36:02,212 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081f8696ecd04ed5aa573f14696dfd62, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732664154842 2024-11-26T23:36:02,212 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a30e8ef23c243c39ce34e6831369f7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732664157197 2024-11-26T23:36:02,212 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807d1195a9f644a28ec6df9e44f5c3e2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732664157197 2024-11-26T23:36:02,212 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ea4a78abdf74d79a70b041fb629d64a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732664159408 2024-11-26T23:36:02,213 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8512e3c3e484e91be939ac572d08dfb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732664159408 2024-11-26T23:36:02,218 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:36:02,219 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#B#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:36:02,219 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/f9763072bbf7493e9bb142612235ba40 is 50, key is test_row_0/B:col10/1732664159408/Put/seqid=0 2024-11-26T23:36:02,220 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126d1c00cb636a44a0eb1357c803dbee56b_b63a20c833469cedd4c0bebf55ce9018 store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:36:02,222 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126d1c00cb636a44a0eb1357c803dbee56b_b63a20c833469cedd4c0bebf55ce9018, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:36:02,222 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d1c00cb636a44a0eb1357c803dbee56b_b63a20c833469cedd4c0bebf55ce9018 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:36:02,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742421_1597 (size=12949) 2024-11-26T23:36:02,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742422_1598 (size=4469) 2024-11-26T23:36:02,625 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#A#compaction#508 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:36:02,626 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/f9763072bbf7493e9bb142612235ba40 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/f9763072bbf7493e9bb142612235ba40 2024-11-26T23:36:02,626 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b6890e636b184ae29830a03be3d05575 is 175, key is test_row_0/A:col10/1732664159408/Put/seqid=0 2024-11-26T23:36:02,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742423_1599 (size=31903) 2024-11-26T23:36:02,630 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/B of b63a20c833469cedd4c0bebf55ce9018 into f9763072bbf7493e9bb142612235ba40(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:36:02,630 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:36:02,630 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/B, priority=13, startTime=1732664162211; duration=0sec 2024-11-26T23:36:02,630 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T23:36:02,630 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:B 2024-11-26T23:36:02,630 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:36:02,631 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:36:02,631 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1541): b63a20c833469cedd4c0bebf55ce9018/C is initiating minor compaction (all files) 2024-11-26T23:36:02,631 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b63a20c833469cedd4c0bebf55ce9018/C in TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:02,631 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/357ac4f757384390b46faf26d5599e2f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/726ba56a1bbc476a8d7b155396b3c716, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/abb5a945400645f5a2b63a589cdf11a3] into tmpdir=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp, totalSize=36.3 K 2024-11-26T23:36:02,631 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 357ac4f757384390b46faf26d5599e2f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732664154842 2024-11-26T23:36:02,631 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting 726ba56a1bbc476a8d7b155396b3c716, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732664157197 2024-11-26T23:36:02,631 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] compactions.Compactor(225): Compacting abb5a945400645f5a2b63a589cdf11a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732664159408 2024-11-26T23:36:02,637 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b63a20c833469cedd4c0bebf55ce9018#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:36:02,637 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/957a98e16ad4493cb537d8111e11b71c is 50, key is test_row_0/C:col10/1732664159408/Put/seqid=0 2024-11-26T23:36:02,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742424_1600 (size=12949) 2024-11-26T23:36:03,033 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/b6890e636b184ae29830a03be3d05575 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b6890e636b184ae29830a03be3d05575 2024-11-26T23:36:03,036 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/A of b63a20c833469cedd4c0bebf55ce9018 into b6890e636b184ae29830a03be3d05575(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:36:03,036 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:36:03,036 INFO [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/A, priority=13, startTime=1732664162211; duration=0sec 2024-11-26T23:36:03,036 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:36:03,036 DEBUG [RS:0;c707e9ace675:43077-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:A 2024-11-26T23:36:03,043 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/957a98e16ad4493cb537d8111e11b71c as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/957a98e16ad4493cb537d8111e11b71c 2024-11-26T23:36:03,046 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b63a20c833469cedd4c0bebf55ce9018/C of b63a20c833469cedd4c0bebf55ce9018 into 957a98e16ad4493cb537d8111e11b71c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:36:03,046 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b63a20c833469cedd4c0bebf55ce9018: 2024-11-26T23:36:03,046 INFO [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., storeName=b63a20c833469cedd4c0bebf55ce9018/C, priority=13, startTime=1732664162211; duration=0sec 2024-11-26T23:36:03,046 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:36:03,046 DEBUG [RS:0;c707e9ace675:43077-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b63a20c833469cedd4c0bebf55ce9018:C 2024-11-26T23:36:03,913 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-11-26T23:36:03,913 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:36:03,913 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,913 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:03,943 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-11-26T23:36:03,943 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:36:03,943 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,943 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:03,972 INFO [Thread-2330 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2330. 2024-11-26T23:36:03,972 DEBUG [Thread-2330 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T23:36:03,972 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,973 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3504 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3541 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3437 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3508 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3538 2024-11-26T23:36:03,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T23:36:03,973 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:36:03,973 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:36:03,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:03,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:03,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T23:36:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T23:36:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T23:36:03,977 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664163977"}]},"ts":"1732664163977"} 2024-11-26T23:36:03,978 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T23:36:03,978 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T23:36:03,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T23:36:03,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, UNASSIGN}] 2024-11-26T23:36:03,980 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, UNASSIGN 2024-11-26T23:36:03,981 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=CLOSING, regionLocation=c707e9ace675,43077,1732663986534 2024-11-26T23:36:03,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, UNASSIGN because future has completed 2024-11-26T23:36:03,982 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:36:03,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534}] 2024-11-26T23:36:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T23:36:04,134 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:04,134 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:36:04,134 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing b63a20c833469cedd4c0bebf55ce9018, disabling compactions & flushes 2024-11-26T23:36:04,134 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:04,134 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:04,134 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. after waiting 0 ms 2024-11-26T23:36:04,134 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:04,134 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing b63a20c833469cedd4c0bebf55ce9018 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-26T23:36:04,139 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126976c906371f44763937637421f21181b_b63a20c833469cedd4c0bebf55ce9018 is 50, key is test_row_1/A:col10/1732664163912/Put/seqid=0 2024-11-26T23:36:04,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742425_1601 (size=9914) 2024-11-26T23:36:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T23:36:04,542 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:36:04,545 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126976c906371f44763937637421f21181b_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126976c906371f44763937637421f21181b_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:04,546 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/fc20d11113b045f5ab3d99e15e7a15c7, store: [table=TestAcidGuarantees family=A region=b63a20c833469cedd4c0bebf55ce9018] 2024-11-26T23:36:04,546 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/fc20d11113b045f5ab3d99e15e7a15c7 is 175, key is test_row_1/A:col10/1732664163912/Put/seqid=0 2024-11-26T23:36:04,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742426_1602 (size=22561) 2024-11-26T23:36:04,597 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:36:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T23:36:04,950 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/fc20d11113b045f5ab3d99e15e7a15c7 2024-11-26T23:36:04,955 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/2bffb44666e141299ff768c625025673 is 50, key is test_row_1/B:col10/1732664163912/Put/seqid=0 2024-11-26T23:36:04,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742427_1603 (size=9857) 2024-11-26T23:36:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T23:36:05,358 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/2bffb44666e141299ff768c625025673 2024-11-26T23:36:05,363 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/a8ffd123fd1a479d895e8337084a6858 is 50, key is test_row_1/C:col10/1732664163912/Put/seqid=0 2024-11-26T23:36:05,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742428_1604 (size=9857) 2024-11-26T23:36:05,767 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/a8ffd123fd1a479d895e8337084a6858 2024-11-26T23:36:05,770 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/A/fc20d11113b045f5ab3d99e15e7a15c7 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/fc20d11113b045f5ab3d99e15e7a15c7 2024-11-26T23:36:05,773 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/fc20d11113b045f5ab3d99e15e7a15c7, entries=100, sequenceid=287, filesize=22.0 K 2024-11-26T23:36:05,773 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/B/2bffb44666e141299ff768c625025673 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2bffb44666e141299ff768c625025673 2024-11-26T23:36:05,776 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2bffb44666e141299ff768c625025673, entries=100, sequenceid=287, filesize=9.6 K 2024-11-26T23:36:05,776 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/.tmp/C/a8ffd123fd1a479d895e8337084a6858 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/a8ffd123fd1a479d895e8337084a6858 2024-11-26T23:36:05,779 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/a8ffd123fd1a479d895e8337084a6858, entries=100, sequenceid=287, filesize=9.6 K 2024-11-26T23:36:05,779 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b63a20c833469cedd4c0bebf55ce9018 in 1645ms, sequenceid=287, compaction requested=false 2024-11-26T23:36:05,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/85c71426bd2f459fbb1a449adc27ee51, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a3ec23e503a8492785204cd83decd2ee, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/e151e6ecc11a4b6eb3fa5ea54807ec5a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/5f09e6516edb4e0089dd55f648a995d5, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b2f7626054174a5a822bb4578d673d49, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a4717fb575ab4fffaf6f626130d1dd05, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/723815347eb74232a94403db8ae385bd, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a01581631306430a903c4ea542d58cfa, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/0438b97e10464ee89e19aa9ca7ddf0c9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/c99258baa51c41ee829856e3946dded7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/8cdc1031dd544d769ab4688cfd416f92, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/6c3eaf48029d4dbba76c947b48e10db4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a5af32bab9954f6f96ed36842b4f1646, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/dc07f6b12a7748f2a9521ceb1898864d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/d00c95034f4c4e6e828461d89731fc2e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/255d0a96bdd64a5aa47f9c0e32c941d0, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/081f8696ecd04ed5aa573f14696dfd62, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/807d1195a9f644a28ec6df9e44f5c3e2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b8512e3c3e484e91be939ac572d08dfb] to archive 2024-11-26T23:36:05,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:36:05,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/85c71426bd2f459fbb1a449adc27ee51 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/85c71426bd2f459fbb1a449adc27ee51 2024-11-26T23:36:05,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a3ec23e503a8492785204cd83decd2ee to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a3ec23e503a8492785204cd83decd2ee 2024-11-26T23:36:05,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/e151e6ecc11a4b6eb3fa5ea54807ec5a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/e151e6ecc11a4b6eb3fa5ea54807ec5a 2024-11-26T23:36:05,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/5f09e6516edb4e0089dd55f648a995d5 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/5f09e6516edb4e0089dd55f648a995d5 2024-11-26T23:36:05,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b2f7626054174a5a822bb4578d673d49 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b2f7626054174a5a822bb4578d673d49 2024-11-26T23:36:05,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a4717fb575ab4fffaf6f626130d1dd05 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a4717fb575ab4fffaf6f626130d1dd05 2024-11-26T23:36:05,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/723815347eb74232a94403db8ae385bd to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/723815347eb74232a94403db8ae385bd 2024-11-26T23:36:05,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a01581631306430a903c4ea542d58cfa to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a01581631306430a903c4ea542d58cfa 2024-11-26T23:36:05,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/0438b97e10464ee89e19aa9ca7ddf0c9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/0438b97e10464ee89e19aa9ca7ddf0c9 2024-11-26T23:36:05,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/c99258baa51c41ee829856e3946dded7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/c99258baa51c41ee829856e3946dded7 2024-11-26T23:36:05,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/8cdc1031dd544d769ab4688cfd416f92 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/8cdc1031dd544d769ab4688cfd416f92 2024-11-26T23:36:05,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/6c3eaf48029d4dbba76c947b48e10db4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/6c3eaf48029d4dbba76c947b48e10db4 2024-11-26T23:36:05,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a5af32bab9954f6f96ed36842b4f1646 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/a5af32bab9954f6f96ed36842b4f1646 2024-11-26T23:36:05,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/dc07f6b12a7748f2a9521ceb1898864d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/dc07f6b12a7748f2a9521ceb1898864d 2024-11-26T23:36:05,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/d00c95034f4c4e6e828461d89731fc2e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/d00c95034f4c4e6e828461d89731fc2e 2024-11-26T23:36:05,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/255d0a96bdd64a5aa47f9c0e32c941d0 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/255d0a96bdd64a5aa47f9c0e32c941d0 2024-11-26T23:36:05,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/081f8696ecd04ed5aa573f14696dfd62 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/081f8696ecd04ed5aa573f14696dfd62 2024-11-26T23:36:05,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/807d1195a9f644a28ec6df9e44f5c3e2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/807d1195a9f644a28ec6df9e44f5c3e2 2024-11-26T23:36:05,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b8512e3c3e484e91be939ac572d08dfb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b8512e3c3e484e91be939ac572d08dfb 2024-11-26T23:36:05,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/228af8af232f43019da77182d83e719a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/445db6b0641846d3b4b992c026f26dbb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/aff46f0570b5442998cf4bb851458b6a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/84e4b943cc2f4c74a3bec4d098f2a9e2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/cf04d6853fe7405a8aa5ba9584176154, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/ab28bc83f8f34e209a390dd020307912, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/e6daf7a608fd44b5b48dccea21f1ff4a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/91a64d5c96c44ce0aa8e8e24f9b2cde1, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c19752426f804aff8c7172f4461d931a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c0390ecd4a1e4fcca7a6466545aedb8b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8f1a2670b6d6461fa188c0deda728497, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c34675c15edb40bba769965104b7cbf9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5b79cfda99d44326b4b5bbd93f972706, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/83e6fcd414bd45309caeb201efbbaf57, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/3f97e4ca016c4b69ab61d65346d1c39a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2fc555cf9fc54b71a7e6f2cd820b00f9, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8a30e8ef23c243c39ce34e6831369f7d, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5ea4a78abdf74d79a70b041fb629d64a] to archive 2024-11-26T23:36:05,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:36:05,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/228af8af232f43019da77182d83e719a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/228af8af232f43019da77182d83e719a 2024-11-26T23:36:05,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/445db6b0641846d3b4b992c026f26dbb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/445db6b0641846d3b4b992c026f26dbb 2024-11-26T23:36:05,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/aff46f0570b5442998cf4bb851458b6a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/aff46f0570b5442998cf4bb851458b6a 2024-11-26T23:36:05,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/7b3c43ff3d1148ce9dcaeea16ee9ed8d 2024-11-26T23:36:05,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/84e4b943cc2f4c74a3bec4d098f2a9e2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/84e4b943cc2f4c74a3bec4d098f2a9e2 2024-11-26T23:36:05,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/cf04d6853fe7405a8aa5ba9584176154 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/cf04d6853fe7405a8aa5ba9584176154 2024-11-26T23:36:05,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/ab28bc83f8f34e209a390dd020307912 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/ab28bc83f8f34e209a390dd020307912 2024-11-26T23:36:05,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/e6daf7a608fd44b5b48dccea21f1ff4a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/e6daf7a608fd44b5b48dccea21f1ff4a 2024-11-26T23:36:05,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/91a64d5c96c44ce0aa8e8e24f9b2cde1 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/91a64d5c96c44ce0aa8e8e24f9b2cde1 2024-11-26T23:36:05,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c19752426f804aff8c7172f4461d931a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c19752426f804aff8c7172f4461d931a 2024-11-26T23:36:05,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c0390ecd4a1e4fcca7a6466545aedb8b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c0390ecd4a1e4fcca7a6466545aedb8b 2024-11-26T23:36:05,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8f1a2670b6d6461fa188c0deda728497 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8f1a2670b6d6461fa188c0deda728497 2024-11-26T23:36:05,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c34675c15edb40bba769965104b7cbf9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/c34675c15edb40bba769965104b7cbf9 2024-11-26T23:36:05,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5b79cfda99d44326b4b5bbd93f972706 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5b79cfda99d44326b4b5bbd93f972706 2024-11-26T23:36:05,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/83e6fcd414bd45309caeb201efbbaf57 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/83e6fcd414bd45309caeb201efbbaf57 2024-11-26T23:36:05,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/3f97e4ca016c4b69ab61d65346d1c39a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/3f97e4ca016c4b69ab61d65346d1c39a 2024-11-26T23:36:05,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2fc555cf9fc54b71a7e6f2cd820b00f9 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2fc555cf9fc54b71a7e6f2cd820b00f9 2024-11-26T23:36:05,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8a30e8ef23c243c39ce34e6831369f7d to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/8a30e8ef23c243c39ce34e6831369f7d 2024-11-26T23:36:05,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5ea4a78abdf74d79a70b041fb629d64a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/5ea4a78abdf74d79a70b041fb629d64a 2024-11-26T23:36:05,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/e2d7b5c268ce43b98abda0e3ef9b2395, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1faf9480b571484db5b8b15ef8277857, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/99e31eda8b5e4964bff64f862d256139, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/3506d92e7f2849e4b3ec38241e41614b, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1d2bfc588cbe404fa93abeb6cd76ec2a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b419b5f0e79c45479d805029e15021c6, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/cc4b0da902314a4ca4602391316adf40, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/f3496f9e86f0461f92e8e8ae7bc72003, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/20c01adb2f124f68b0f3cc17184405e7, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/51f27e83893542d08b19e65e1927c89a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b3d2efd5d4b1487c9144d392ffbd447e, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/71cd1be639164918bdead4088bd628d4, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/d0ea2f58c1194d6981e7390a159e2f3a, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/10227e9595964402bcac11866c38e224, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/46d8eaddf7114480b12a39d04b6c85c2, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/357ac4f757384390b46faf26d5599e2f, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/528f194acb7e4dfc9025648eb7ed2aeb, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/726ba56a1bbc476a8d7b155396b3c716, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/abb5a945400645f5a2b63a589cdf11a3] to archive 2024-11-26T23:36:05,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:36:05,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/e2d7b5c268ce43b98abda0e3ef9b2395 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/e2d7b5c268ce43b98abda0e3ef9b2395 2024-11-26T23:36:05,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1faf9480b571484db5b8b15ef8277857 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1faf9480b571484db5b8b15ef8277857 2024-11-26T23:36:05,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/99e31eda8b5e4964bff64f862d256139 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/99e31eda8b5e4964bff64f862d256139 2024-11-26T23:36:05,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/3506d92e7f2849e4b3ec38241e41614b to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/3506d92e7f2849e4b3ec38241e41614b 2024-11-26T23:36:05,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1d2bfc588cbe404fa93abeb6cd76ec2a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/1d2bfc588cbe404fa93abeb6cd76ec2a 2024-11-26T23:36:05,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b419b5f0e79c45479d805029e15021c6 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b419b5f0e79c45479d805029e15021c6 2024-11-26T23:36:05,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/cc4b0da902314a4ca4602391316adf40 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/cc4b0da902314a4ca4602391316adf40 2024-11-26T23:36:05,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/f3496f9e86f0461f92e8e8ae7bc72003 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/f3496f9e86f0461f92e8e8ae7bc72003 2024-11-26T23:36:05,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/20c01adb2f124f68b0f3cc17184405e7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/20c01adb2f124f68b0f3cc17184405e7 2024-11-26T23:36:05,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/51f27e83893542d08b19e65e1927c89a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/51f27e83893542d08b19e65e1927c89a 2024-11-26T23:36:05,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b3d2efd5d4b1487c9144d392ffbd447e to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/b3d2efd5d4b1487c9144d392ffbd447e 2024-11-26T23:36:05,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/71cd1be639164918bdead4088bd628d4 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/71cd1be639164918bdead4088bd628d4 2024-11-26T23:36:05,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/d0ea2f58c1194d6981e7390a159e2f3a to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/d0ea2f58c1194d6981e7390a159e2f3a 2024-11-26T23:36:05,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/10227e9595964402bcac11866c38e224 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/10227e9595964402bcac11866c38e224 2024-11-26T23:36:05,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/46d8eaddf7114480b12a39d04b6c85c2 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/46d8eaddf7114480b12a39d04b6c85c2 2024-11-26T23:36:05,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/357ac4f757384390b46faf26d5599e2f to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/357ac4f757384390b46faf26d5599e2f 2024-11-26T23:36:05,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/528f194acb7e4dfc9025648eb7ed2aeb to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/528f194acb7e4dfc9025648eb7ed2aeb 2024-11-26T23:36:05,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/726ba56a1bbc476a8d7b155396b3c716 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/726ba56a1bbc476a8d7b155396b3c716 2024-11-26T23:36:05,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/abb5a945400645f5a2b63a589cdf11a3 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/abb5a945400645f5a2b63a589cdf11a3 2024-11-26T23:36:05,831 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/recovered.edits/290.seqid, newMaxSeqId=290, maxSeqId=4 2024-11-26T23:36:05,832 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. 2024-11-26T23:36:05,832 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for b63a20c833469cedd4c0bebf55ce9018: Waiting for close lock at 1732664164134Running coprocessor pre-close hooks at 1732664164134Disabling compacts and flushes for region at 1732664164134Disabling writes for close at 1732664164134Obtaining lock to block concurrent updates at 1732664164134Preparing flush snapshotting stores in b63a20c833469cedd4c0bebf55ce9018 at 1732664164134Finished memstore snapshotting TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732664164135 (+1 ms)Flushing stores of TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018. at 1732664164135Flushing b63a20c833469cedd4c0bebf55ce9018/A: creating writer at 1732664164135Flushing mob file b63a20c833469cedd4c0bebf55ce9018/A: appending metadata at 1732664164139 (+4 ms)Flushing mob file b63a20c833469cedd4c0bebf55ce9018/A: closing flushed file at 1732664164139Flushing b63a20c833469cedd4c0bebf55ce9018/A: appending metadata at 1732664164546 (+407 ms)Flushing b63a20c833469cedd4c0bebf55ce9018/A: closing flushed file at 1732664164546Flushing b63a20c833469cedd4c0bebf55ce9018/B: creating writer at 1732664164953 (+407 ms)Flushing b63a20c833469cedd4c0bebf55ce9018/B: appending metadata at 1732664164954 (+1 ms)Flushing b63a20c833469cedd4c0bebf55ce9018/B: closing flushed file at 1732664164954Flushing b63a20c833469cedd4c0bebf55ce9018/C: creating writer at 1732664165361 (+407 ms)Flushing b63a20c833469cedd4c0bebf55ce9018/C: appending metadata at 1732664165363 (+2 ms)Flushing b63a20c833469cedd4c0bebf55ce9018/C: closing flushed file at 1732664165363Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ab53e86: reopening flushed file at 1732664165770 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1437a650: reopening flushed file at 1732664165773 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@718a4658: reopening flushed file at 1732664165776 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b63a20c833469cedd4c0bebf55ce9018 in 1645ms, sequenceid=287, compaction requested=false at 1732664165779 (+3 ms)Writing region close event to WAL at 1732664165829 (+50 ms)Running coprocessor post-close hooks at 1732664165831 (+2 ms)Closed at 1732664165832 (+1 ms) 2024-11-26T23:36:05,833 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:05,834 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=b63a20c833469cedd4c0bebf55ce9018, regionState=CLOSED 2024-11-26T23:36:05,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 because future has completed 2024-11-26T23:36:05,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-26T23:36:05,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure b63a20c833469cedd4c0bebf55ce9018, server=c707e9ace675,43077,1732663986534 in 1.8530 sec 2024-11-26T23:36:05,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-26T23:36:05,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b63a20c833469cedd4c0bebf55ce9018, UNASSIGN in 1.8570 sec 2024-11-26T23:36:05,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-26T23:36:05,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8600 sec 2024-11-26T23:36:05,841 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732664165841"}]},"ts":"1732664165841"} 2024-11-26T23:36:05,842 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T23:36:05,842 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T23:36:05,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8680 sec 2024-11-26T23:36:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T23:36:06,122 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:36:06,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T23:36:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:36:06,124 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:36:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-26T23:36:06,124 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:36:06,126 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,127 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C, FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/recovered.edits] 2024-11-26T23:36:06,129 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b6890e636b184ae29830a03be3d05575 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/b6890e636b184ae29830a03be3d05575 2024-11-26T23:36:06,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/fc20d11113b045f5ab3d99e15e7a15c7 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/A/fc20d11113b045f5ab3d99e15e7a15c7 2024-11-26T23:36:06,131 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2bffb44666e141299ff768c625025673 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/2bffb44666e141299ff768c625025673 2024-11-26T23:36:06,132 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/f9763072bbf7493e9bb142612235ba40 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/B/f9763072bbf7493e9bb142612235ba40 2024-11-26T23:36:06,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/957a98e16ad4493cb537d8111e11b71c to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/957a98e16ad4493cb537d8111e11b71c 2024-11-26T23:36:06,135 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/a8ffd123fd1a479d895e8337084a6858 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/C/a8ffd123fd1a479d895e8337084a6858 2024-11-26T23:36:06,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/recovered.edits/290.seqid to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018/recovered.edits/290.seqid 2024-11-26T23:36:06,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/default/TestAcidGuarantees/b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,137 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T23:36:06,137 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T23:36:06,138 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T23:36:06,140 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112612766a1da56544c994770a7cab6d1496_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112612766a1da56544c994770a7cab6d1496_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,141 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261ca8d2d3254641fc8795fca5fe6c6f3d_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261ca8d2d3254641fc8795fca5fe6c6f3d_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,141 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261d698fc3610d4e3ab963bbff480008cb_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261d698fc3610d4e3ab963bbff480008cb_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,142 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112620e5f7e76a6743479db78b580f24030c_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112620e5f7e76a6743479db78b580f24030c_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,143 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263d4d53e8c61042019da12dd14e50b995_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263d4d53e8c61042019da12dd14e50b995_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T23:36:06,144 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265c70f6c8791e46b69e67aefe0066beda_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265c70f6c8791e46b69e67aefe0066beda_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,145 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f7b765e61f14a03a1cf32a119964a57_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f7b765e61f14a03a1cf32a119964a57_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,146 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126643f3e433ff64c37b135a81f731be59b_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126643f3e433ff64c37b135a81f731be59b_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,147 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267130b78e68ee4924b354780728044e47_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267130b78e68ee4924b354780728044e47_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,148 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112674ff3ff941264a5cb38d812039340bf1_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112674ff3ff941264a5cb38d812039340bf1_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,148 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112679c3a8fc42fa477e9ef2ee97715d0921_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112679c3a8fc42fa477e9ef2ee97715d0921_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,149 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126976c906371f44763937637421f21181b_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126976c906371f44763937637421f21181b_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,150 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a1514b466db44b01a4dfefd92d809ca7_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a1514b466db44b01a4dfefd92d809ca7_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,151 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bd8ae94fa30d46869e207bf4efe881d0_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bd8ae94fa30d46869e207bf4efe881d0_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,152 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e37dfedace3743bf9de56c96687f25ff_b63a20c833469cedd4c0bebf55ce9018 to hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e37dfedace3743bf9de56c96687f25ff_b63a20c833469cedd4c0bebf55ce9018 2024-11-26T23:36:06,152 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T23:36:06,154 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:36:06,156 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T23:36:06,158 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T23:36:06,159 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:36:06,159 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T23:36:06,159 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732664166159"}]},"ts":"9223372036854775807"} 2024-11-26T23:36:06,161 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T23:36:06,161 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b63a20c833469cedd4c0bebf55ce9018, NAME => 'TestAcidGuarantees,,1732664136870.b63a20c833469cedd4c0bebf55ce9018.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T23:36:06,161 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T23:36:06,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732664166161"}]},"ts":"9223372036854775807"} 2024-11-26T23:36:06,162 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T23:36:06,162 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T23:36:06,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-26T23:36:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-26T23:36:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T23:36:06,232 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T23:36:06,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 242), OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=6520 (was 6519) - AvailableMemoryMB LEAK? - 2024-11-26T23:36:06,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:36:06,241 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:36:06,241 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:36:06,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:06,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:06,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:36:06,241 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:36:06,241 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-11-26T23:36:06,242 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,35833,1732663985624 2024-11-26T23:36:06,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:36:06,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:36:06,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:36:06,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:36:06,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:36:06,244 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:36:06,244 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:36:06,245 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:36:06,245 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:36:06,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:06,245 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,43077,1732663986534' ***** 2024-11-26T23:36:06,245 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:36:06,245 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:36:06,246 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,43077,1732663986534 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:43077. 2024-11-26T23:36:06,246 DEBUG [RS:0;c707e9ace675:43077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:36:06,246 DEBUG [RS:0;c707e9ace675:43077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:36:06,246 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:36:06,247 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T23:36:06,247 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T23:36:06,247 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:06,247 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:36:06,247 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:36:06,247 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:36:06,247 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:36:06,247 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:36:06,248 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-26T23:36:06,270 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/info/69ed5a1fab1f4f85aa8905b63c3a5c60 is 95, key is TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab./info:/1732664018275/DeleteFamily/seqid=0 2024-11-26T23:36:06,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742429_1605 (size=6514) 2024-11-26T23:36:06,281 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:36:06,448 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:06,648 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:06,674 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/info/69ed5a1fab1f4f85aa8905b63c3a5c60 2024-11-26T23:36:06,691 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/ns/f26a892c4d4e46e29a3c0107425a4b9e is 93, key is TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab./ns:/1732664018230/DeleteFamily/seqid=0 2024-11-26T23:36:06,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742430_1606 (size=5906) 2024-11-26T23:36:06,848 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:07,048 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:07,077 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T23:36:07,077 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T23:36:07,095 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/ns/f26a892c4d4e46e29a3c0107425a4b9e 2024-11-26T23:36:07,112 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/rep_barrier/edd8ecdb3446468d9bf586deb9f64f31 is 102, key is TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab./rep_barrier:/1732664018230/DeleteFamily/seqid=0 2024-11-26T23:36:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742431_1607 (size=6025) 2024-11-26T23:36:07,248 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:07,448 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:07,515 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/rep_barrier/edd8ecdb3446468d9bf586deb9f64f31 2024-11-26T23:36:07,533 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/table/cae51b9eced04dc287303c34dbc806d9 is 96, key is TestAcidGuarantees,,1732663990642.56adae88c925ec88c99616a57d431fab./table:/1732664018230/DeleteFamily/seqid=0 2024-11-26T23:36:07,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742432_1608 (size=5936) 2024-11-26T23:36:07,649 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:07,849 DEBUG [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:36:07,937 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/table/cae51b9eced04dc287303c34dbc806d9 2024-11-26T23:36:07,940 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/info/69ed5a1fab1f4f85aa8905b63c3a5c60 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/info/69ed5a1fab1f4f85aa8905b63c3a5c60 2024-11-26T23:36:07,943 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/info/69ed5a1fab1f4f85aa8905b63c3a5c60, entries=12, sequenceid=90, filesize=6.4 K 2024-11-26T23:36:07,943 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/ns/f26a892c4d4e46e29a3c0107425a4b9e as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/ns/f26a892c4d4e46e29a3c0107425a4b9e 2024-11-26T23:36:07,946 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/ns/f26a892c4d4e46e29a3c0107425a4b9e, entries=8, sequenceid=90, filesize=5.8 K 2024-11-26T23:36:07,947 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/rep_barrier/edd8ecdb3446468d9bf586deb9f64f31 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/rep_barrier/edd8ecdb3446468d9bf586deb9f64f31 2024-11-26T23:36:07,949 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/rep_barrier/edd8ecdb3446468d9bf586deb9f64f31, entries=6, sequenceid=90, filesize=5.9 K 2024-11-26T23:36:07,950 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/.tmp/table/cae51b9eced04dc287303c34dbc806d9 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/table/cae51b9eced04dc287303c34dbc806d9 2024-11-26T23:36:07,952 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/table/cae51b9eced04dc287303c34dbc806d9, entries=7, sequenceid=90, filesize=5.8 K 2024-11-26T23:36:07,953 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-11-26T23:36:07,956 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-26T23:36:07,957 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:36:07,957 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:36:07,957 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732664166247Running coprocessor pre-close hooks at 1732664166247Disabling compacts and flushes for region at 1732664166247Disabling writes for close at 1732664166247Obtaining lock to block concurrent updates at 1732664166248 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732664166248Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732664166248Flushing stores of hbase:meta,,1.1588230740 at 1732664166248Flushing 1588230740/info: creating writer at 1732664166248Flushing 1588230740/info: appending metadata at 1732664166270 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732664166270Flushing 1588230740/ns: creating writer at 1732664166677 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732664166691 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732664166691Flushing 1588230740/rep_barrier: creating writer at 1732664167098 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732664167111 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732664167111Flushing 1588230740/table: creating writer at 1732664167518 (+407 ms)Flushing 1588230740/table: appending metadata at 1732664167533 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732664167533Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c44e0b: reopening flushed file at 1732664167940 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c8cb09: reopening flushed file at 1732664167943 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734d7ca5: reopening flushed file at 1732664167946 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55dd614e: reopening flushed file at 1732664167949 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1732664167953 (+4 ms)Writing region close event to WAL at 1732664167954 (+1 ms)Running coprocessor post-close hooks at 1732664167957 (+3 ms)Closed at 1732664167957 2024-11-26T23:36:07,957 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:36:08,049 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,43077,1732663986534; all regions closed. 2024-11-26T23:36:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741834_1010 (size=25344) 2024-11-26T23:36:08,055 DEBUG [RS:0;c707e9ace675:43077 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/oldWALs 2024-11-26T23:36:08,055 INFO [RS:0;c707e9ace675:43077 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c707e9ace675%2C43077%2C1732663986534.meta:.meta(num 1732663990175) 2024-11-26T23:36:08,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741832_1008 (size=13828136) 2024-11-26T23:36:08,058 DEBUG [RS:0;c707e9ace675:43077 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/oldWALs 2024-11-26T23:36:08,059 INFO [RS:0;c707e9ace675:43077 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c707e9ace675%2C43077%2C1732663986534:(num 1732663989172) 2024-11-26T23:36:08,059 DEBUG [RS:0;c707e9ace675:43077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:36:08,059 INFO [RS:0;c707e9ace675:43077 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:36:08,059 INFO [RS:0;c707e9ace675:43077 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:36:08,059 INFO [RS:0;c707e9ace675:43077 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:36:08,059 INFO [RS:0;c707e9ace675:43077 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:36:08,059 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:36:08,059 INFO [RS:0;c707e9ace675:43077 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43077 2024-11-26T23:36:08,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,43077,1732663986534 2024-11-26T23:36:08,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:36:08,063 INFO [RS:0;c707e9ace675:43077 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:36:08,065 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,43077,1732663986534] 2024-11-26T23:36:08,067 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,43077,1732663986534 already deleted, retry=false 2024-11-26T23:36:08,067 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,43077,1732663986534 expired; onlineServers=0 2024-11-26T23:36:08,067 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,35833,1732663985624' ***** 2024-11-26T23:36:08,067 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:36:08,067 INFO [M:0;c707e9ace675:35833 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:36:08,067 INFO [M:0;c707e9ace675:35833 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:36:08,067 DEBUG [M:0;c707e9ace675:35833 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:36:08,068 DEBUG [M:0;c707e9ace675:35833 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:36:08,068 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663988902 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663988902,5,FailOnTimeoutGroup] 2024-11-26T23:36:08,068 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663988903 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663988903,5,FailOnTimeoutGroup] 2024-11-26T23:36:08,068 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:36:08,068 INFO [M:0;c707e9ace675:35833 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:36:08,068 INFO [M:0;c707e9ace675:35833 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:36:08,068 DEBUG [M:0;c707e9ace675:35833 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:36:08,068 INFO [M:0;c707e9ace675:35833 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:36:08,068 INFO [M:0;c707e9ace675:35833 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:36:08,068 ERROR [M:0;c707e9ace675:35833 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-26T23:36:08,069 INFO [M:0;c707e9ace675:35833 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:36:08,069 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:36:08,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:36:08,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:36:08,070 DEBUG [M:0;c707e9ace675:35833 {}] zookeeper.ZKUtil(347): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:36:08,070 WARN [M:0;c707e9ace675:35833 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:36:08,070 INFO [M:0;c707e9ace675:35833 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/.lastflushedseqids 2024-11-26T23:36:08,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742433_1609 (size=72) 2024-11-26T23:36:08,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:36:08,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43077-0x10156a02cac0001, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:36:08,166 INFO [RS:0;c707e9ace675:43077 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:36:08,166 INFO [RS:0;c707e9ace675:43077 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,43077,1732663986534; zookeeper connection closed. 2024-11-26T23:36:08,166 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2cf742e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2cf742e7 2024-11-26T23:36:08,167 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T23:36:08,480 INFO [M:0;c707e9ace675:35833 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:36:08,480 INFO [M:0;c707e9ace675:35833 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:36:08,480 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:36:08,480 INFO [M:0;c707e9ace675:35833 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:36:08,480 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:36:08,480 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:36:08,481 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:36:08,481 INFO [M:0;c707e9ace675:35833 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=751.63 KB heapSize=923.22 KB 2024-11-26T23:36:08,496 DEBUG [M:0;c707e9ace675:35833 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4e31d993e06143e29de0f333f8cd11a1 is 82, key is hbase:meta,,1/info:regioninfo/1732663990269/Put/seqid=0 2024-11-26T23:36:08,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742434_1610 (size=5672) 2024-11-26T23:36:08,899 INFO [M:0;c707e9ace675:35833 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4e31d993e06143e29de0f333f8cd11a1 2024-11-26T23:36:08,919 DEBUG [M:0;c707e9ace675:35833 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/03472aa439ac45c5ada2eaad2f03ab81 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1732664139933/Put/seqid=0 2024-11-26T23:36:08,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742435_1611 (size=43502) 2024-11-26T23:36:09,323 INFO [M:0;c707e9ace675:35833 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=751.08 KB at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/03472aa439ac45c5ada2eaad2f03ab81 2024-11-26T23:36:09,326 INFO [M:0;c707e9ace675:35833 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03472aa439ac45c5ada2eaad2f03ab81 2024-11-26T23:36:09,340 DEBUG [M:0;c707e9ace675:35833 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e746ce91de164c7cb60cb49387a7ff24 is 69, key is c707e9ace675,43077,1732663986534/rs:state/1732663988939/Put/seqid=0 2024-11-26T23:36:09,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073742436_1612 (size=5156) 2024-11-26T23:36:09,744 INFO [M:0;c707e9ace675:35833 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e746ce91de164c7cb60cb49387a7ff24 2024-11-26T23:36:09,747 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4e31d993e06143e29de0f333f8cd11a1 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4e31d993e06143e29de0f333f8cd11a1 2024-11-26T23:36:09,750 INFO [M:0;c707e9ace675:35833 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4e31d993e06143e29de0f333f8cd11a1, entries=8, sequenceid=2117, filesize=5.5 K 2024-11-26T23:36:09,751 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/03472aa439ac45c5ada2eaad2f03ab81 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/03472aa439ac45c5ada2eaad2f03ab81 2024-11-26T23:36:09,753 INFO [M:0;c707e9ace675:35833 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03472aa439ac45c5ada2eaad2f03ab81 2024-11-26T23:36:09,753 INFO [M:0;c707e9ace675:35833 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/03472aa439ac45c5ada2eaad2f03ab81, entries=166, sequenceid=2117, filesize=42.5 K 2024-11-26T23:36:09,754 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e746ce91de164c7cb60cb49387a7ff24 as hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e746ce91de164c7cb60cb49387a7ff24 2024-11-26T23:36:09,756 INFO [M:0;c707e9ace675:35833 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36065/user/jenkins/test-data/a334252f-fe96-7cf7-1329-30feb77d31fd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e746ce91de164c7cb60cb49387a7ff24, entries=1, sequenceid=2117, filesize=5.0 K 2024-11-26T23:36:09,757 INFO [M:0;c707e9ace675:35833 {}] regionserver.HRegion(3140): Finished flush of dataSize ~751.63 KB/769671, heapSize ~922.92 KB/945072, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2117, compaction requested=false 2024-11-26T23:36:09,758 INFO [M:0;c707e9ace675:35833 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:36:09,758 DEBUG [M:0;c707e9ace675:35833 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732664168480Disabling compacts and flushes for region at 1732664168480Disabling writes for close at 1732664168481 (+1 ms)Obtaining lock to block concurrent updates at 1732664168481Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732664168481Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=769671, getHeapSize=945312, getOffHeapSize=0, getCellsCount=2308 at 1732664168481Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732664168481Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732664168481Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732664168496 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732664168496Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732664168903 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732664168918 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732664168919 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732664169326 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732664169340 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732664169340Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6dca17: reopening flushed file at 1732664169747 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c1db7f6: reopening flushed file at 1732664169750 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5839b91d: reopening flushed file at 1732664169753 (+3 ms)Finished flush of dataSize ~751.63 KB/769671, heapSize ~922.92 KB/945072, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2117, compaction requested=false at 1732664169757 (+4 ms)Writing region close event to WAL at 1732664169758 (+1 ms)Closed at 1732664169758 2024-11-26T23:36:09,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36035 is added to blk_1073741830_1006 (size=908459) 2024-11-26T23:36:09,763 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:36:09,763 INFO [M:0;c707e9ace675:35833 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:36:09,764 INFO [M:0;c707e9ace675:35833 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35833 2024-11-26T23:36:09,764 INFO [M:0;c707e9ace675:35833 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:36:09,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:36:09,866 INFO [M:0;c707e9ace675:35833 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:36:09,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35833-0x10156a02cac0000, quorum=127.0.0.1:60864, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:36:09,870 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:36:09,873 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:36:09,873 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:36:09,873 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:36:09,873 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/hadoop.log.dir/,STOPPED} 2024-11-26T23:36:09,876 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:36:09,876 WARN [BP-1290680769-172.17.0.2-1732663982490 heartbeating to localhost/127.0.0.1:36065 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:36:09,876 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:36:09,876 WARN [BP-1290680769-172.17.0.2-1732663982490 heartbeating to localhost/127.0.0.1:36065 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1290680769-172.17.0.2-1732663982490 (Datanode Uuid 6dcbde4b-1ee6-4b73-8737-d9c1a29e5185) service to localhost/127.0.0.1:36065 2024-11-26T23:36:09,879 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0/data/data1/current/BP-1290680769-172.17.0.2-1732663982490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:36:09,879 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/cluster_0d69331c-abd3-3b9f-94ab-ddff835e86f0/data/data2/current/BP-1290680769-172.17.0.2-1732663982490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:36:09,880 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:36:09,886 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:36:09,887 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:36:09,887 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:36:09,887 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:36:09,887 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/01ccf58a-97fa-b889-816b-617b2bf0c0f0/hadoop.log.dir/,STOPPED} 2024-11-26T23:36:09,902 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:36:10,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down